diff options
| author | Chocobozzz <me@florianbigard.com> | 2023-07-31 14:34:36 +0200 |
|---|---|---|
| committer | Chocobozzz <me@florianbigard.com> | 2023-08-11 15:02:33 +0200 |
| commit | 3a4992633ee62d5edfbb484d9c6bcb3cf158489d (patch) | |
| tree | e4510b39bdac9c318fdb4b47018d08f15368b8f0 /server/lib | |
| parent | 04d1da5621d25d59bd5fa1543b725c497bf5d9a8 (diff) | |
| download | PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.gz PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.zst PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.zip | |
Migrate server to ESM
Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:
* Server can be faster at startup because imports() are async and we can
easily lazy import big modules
* Angular doesn't seem to support ES import (with .js extension), so we
had to correctly organize peertube into a monorepo:
* Use yarn workspace feature
* Use typescript reference projects for dependencies
* Shared projects have been moved into "packages", each one is now a
node module (with a dedicated package.json/tsconfig.json)
* server/tools have been moved into apps/ and is now a dedicated app
bundled and published on NPM so users don't have to build peertube
cli tools manually
* server/tests have been moved into packages/ so we don't compile
them every time we want to run the server
* Use isolatedModule option:
* Had to move from const enum to const
(https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
* Had to explictely specify "type" imports when used in decorators
* Prefer tsx (that uses esbuild under the hood) instead of ts-node to
load typescript files (tests with mocha or scripts):
* To reduce test complexity as esbuild doesn't support decorator
metadata, we only test server files that do not import server
models
* We still build tests files into js files for a faster CI
* Remove unmaintained peertube CLI import script
* Removed some barrels to speed up execution (less imports)
Diffstat (limited to 'server/lib')
301 files changed, 0 insertions, 27409 deletions
diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts deleted file mode 100644 index 391bcd9c6..000000000 --- a/server/lib/activitypub/activity.ts +++ /dev/null | |||
| @@ -1,74 +0,0 @@ | |||
| 1 | import { doJSONRequest, PeerTubeRequestOptions } from '@server/helpers/requests' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { ActivityObject, ActivityPubActor, ActivityType, APObjectId } from '@shared/models' | ||
| 4 | import { buildSignedRequestOptions } from './send' | ||
| 5 | |||
| 6 | export function getAPId (object: string | { id: string }) { | ||
| 7 | if (typeof object === 'string') return object | ||
| 8 | |||
| 9 | return object.id | ||
| 10 | } | ||
| 11 | |||
| 12 | export function getActivityStreamDuration (duration: number) { | ||
| 13 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
| 14 | return 'PT' + duration + 'S' | ||
| 15 | } | ||
| 16 | |||
| 17 | export function getDurationFromActivityStream (duration: string) { | ||
| 18 | return parseInt(duration.replace(/[^\d]+/, '')) | ||
| 19 | } | ||
| 20 | |||
| 21 | // --------------------------------------------------------------------------- | ||
| 22 | |||
| 23 | export function buildAvailableActivities (): ActivityType[] { | ||
| 24 | return [ | ||
| 25 | 'Create', | ||
| 26 | 'Update', | ||
| 27 | 'Delete', | ||
| 28 | 'Follow', | ||
| 29 | 'Accept', | ||
| 30 | 'Announce', | ||
| 31 | 'Undo', | ||
| 32 | 'Like', | ||
| 33 | 'Reject', | ||
| 34 | 'View', | ||
| 35 | 'Dislike', | ||
| 36 | 'Flag' | ||
| 37 | ] | ||
| 38 | } | ||
| 39 | |||
| 40 | // --------------------------------------------------------------------------- | ||
| 41 | |||
| 42 | export async function fetchAP <T> (url: string, moreOptions: PeerTubeRequestOptions = {}) { | ||
| 43 | const options = { | ||
| 44 | activityPub: true, | ||
| 45 | |||
| 46 | httpSignature: CONFIG.FEDERATION.SIGN_FEDERATED_FETCHES | ||
| 47 | ? await buildSignedRequestOptions({ hasPayload: false }) | ||
| 48 | : undefined, | ||
| 49 | |||
| 50 | ...moreOptions | ||
| 51 | } | ||
| 52 | |||
| 53 | return doJSONRequest<T>(url, options) | ||
| 54 | } | ||
| 55 | |||
| 56 | export async function fetchAPObjectIfNeeded <T extends (ActivityObject | ActivityPubActor)> (object: APObjectId) { | ||
| 57 | if (typeof object === 'string') { | ||
| 58 | const { body } = await fetchAP<Exclude<T, string>>(object) | ||
| 59 | |||
| 60 | return body | ||
| 61 | } | ||
| 62 | |||
| 63 | return object as Exclude<T, string> | ||
| 64 | } | ||
| 65 | |||
| 66 | export async function findLatestAPRedirection (url: string, iteration = 1) { | ||
| 67 | if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) | ||
| 68 | |||
| 69 | const { headers } = await fetchAP(url, { followRedirect: false }) | ||
| 70 | |||
| 71 | if (headers.location) return findLatestAPRedirection(headers.location, iteration + 1) | ||
| 72 | |||
| 73 | return url | ||
| 74 | } | ||
diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts deleted file mode 100644 index dd2bc9f03..000000000 --- a/server/lib/activitypub/actors/get.ts +++ /dev/null | |||
| @@ -1,143 +0,0 @@ | |||
| 1 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { JobQueue } from '@server/lib/job-queue' | ||
| 4 | import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders' | ||
| 5 | import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' | ||
| 6 | import { arrayify } from '@shared/core-utils' | ||
| 7 | import { ActivityPubActor, APObjectId } from '@shared/models' | ||
| 8 | import { fetchAPObjectIfNeeded, getAPId } from '../activity' | ||
| 9 | import { checkUrlsSameHost } from '../url' | ||
| 10 | import { refreshActorIfNeeded } from './refresh' | ||
| 11 | import { APActorCreator, fetchRemoteActor } from './shared' | ||
| 12 | |||
| 13 | function getOrCreateAPActor ( | ||
| 14 | activityActor: string | ActivityPubActor, | ||
| 15 | fetchType: 'all', | ||
| 16 | recurseIfNeeded?: boolean, | ||
| 17 | updateCollections?: boolean | ||
| 18 | ): Promise<MActorFullActor> | ||
| 19 | |||
| 20 | function getOrCreateAPActor ( | ||
| 21 | activityActor: string | ActivityPubActor, | ||
| 22 | fetchType?: 'association-ids', | ||
| 23 | recurseIfNeeded?: boolean, | ||
| 24 | updateCollections?: boolean | ||
| 25 | ): Promise<MActorAccountChannelId> | ||
| 26 | |||
| 27 | async function getOrCreateAPActor ( | ||
| 28 | activityActor: string | ActivityPubActor, | ||
| 29 | fetchType: ActorLoadByUrlType = 'association-ids', | ||
| 30 | recurseIfNeeded = true, | ||
| 31 | updateCollections = false | ||
| 32 | ): Promise<MActorFullActor | MActorAccountChannelId> { | ||
| 33 | const actorUrl = getAPId(activityActor) | ||
| 34 | let actor = await loadActorFromDB(actorUrl, fetchType) | ||
| 35 | |||
| 36 | let created = false | ||
| 37 | let accountPlaylistsUrl: string | ||
| 38 | |||
| 39 | // We don't have this actor in our database, fetch it on remote | ||
| 40 | if (!actor) { | ||
| 41 | const { actorObject } = await fetchRemoteActor(actorUrl) | ||
| 42 | if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) | ||
| 43 | |||
| 44 | // actorUrl is just an alias/redirection, so process object id instead | ||
| 45 | if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections) | ||
| 46 | |||
| 47 | // Create the attributed to actor | ||
| 48 | // In PeerTube a video channel is owned by an account | ||
| 49 | let ownerActor: MActorFullActor | ||
| 50 | if (recurseIfNeeded === true && actorObject.type === 'Group') { | ||
| 51 | ownerActor = await getOrCreateAPOwner(actorObject, actorUrl) | ||
| 52 | } | ||
| 53 | |||
| 54 | const creator = new APActorCreator(actorObject, ownerActor) | ||
| 55 | actor = await retryTransactionWrapper(creator.create.bind(creator)) | ||
| 56 | created = true | ||
| 57 | accountPlaylistsUrl = actorObject.playlists | ||
| 58 | } | ||
| 59 | |||
| 60 | if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor | ||
| 61 | if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor | ||
| 62 | |||
| 63 | const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType }) | ||
| 64 | if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') | ||
| 65 | |||
| 66 | await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) | ||
| 67 | await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl) | ||
| 68 | |||
| 69 | return actorRefreshed | ||
| 70 | } | ||
| 71 | |||
| 72 | async function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { | ||
| 73 | const accountAttributedTo = await findOwner(actorUrl, actorObject.attributedTo, 'Person') | ||
| 74 | if (!accountAttributedTo) { | ||
| 75 | throw new Error(`Cannot find account attributed to video channel ${actorUrl}`) | ||
| 76 | } | ||
| 77 | |||
| 78 | try { | ||
| 79 | // Don't recurse another time | ||
| 80 | const recurseIfNeeded = false | ||
| 81 | return getOrCreateAPActor(accountAttributedTo, 'all', recurseIfNeeded) | ||
| 82 | } catch (err) { | ||
| 83 | logger.error('Cannot get or create account attributed to video channel ' + actorUrl) | ||
| 84 | throw new Error(err) | ||
| 85 | } | ||
| 86 | } | ||
| 87 | |||
| 88 | async function findOwner (rootUrl: string, attributedTo: APObjectId[] | APObjectId, type: 'Person' | 'Group') { | ||
| 89 | for (const actorToCheck of arrayify(attributedTo)) { | ||
| 90 | const actorObject = await fetchAPObjectIfNeeded<ActivityPubActor>(getAPId(actorToCheck)) | ||
| 91 | |||
| 92 | if (!actorObject) { | ||
| 93 | logger.warn('Unknown attributed to actor %s for owner %s', actorToCheck, rootUrl) | ||
| 94 | continue | ||
| 95 | } | ||
| 96 | |||
| 97 | if (checkUrlsSameHost(actorObject.id, rootUrl) !== true) { | ||
| 98 | logger.warn(`Account attributed to ${actorObject.id} does not have the same host than owner actor url ${rootUrl}`) | ||
| 99 | continue | ||
| 100 | } | ||
| 101 | |||
| 102 | if (actorObject.type === type) return actorObject | ||
| 103 | } | ||
| 104 | |||
| 105 | return undefined | ||
| 106 | } | ||
| 107 | |||
| 108 | // --------------------------------------------------------------------------- | ||
| 109 | |||
| 110 | export { | ||
| 111 | getOrCreateAPOwner, | ||
| 112 | getOrCreateAPActor, | ||
| 113 | findOwner | ||
| 114 | } | ||
| 115 | |||
| 116 | // --------------------------------------------------------------------------- | ||
| 117 | |||
| 118 | async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType) { | ||
| 119 | let actor = await loadActorByUrl(actorUrl, fetchType) | ||
| 120 | |||
| 121 | // Orphan actor (not associated to an account of channel) so recreate it | ||
| 122 | if (actor && (!actor.Account && !actor.VideoChannel)) { | ||
| 123 | await actor.destroy() | ||
| 124 | actor = null | ||
| 125 | } | ||
| 126 | |||
| 127 | return actor | ||
| 128 | } | ||
| 129 | |||
| 130 | async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) { | ||
| 131 | if ((created === true || refreshed === true) && updateCollections === true) { | ||
| 132 | const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } | ||
| 133 | await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | ||
| 134 | } | ||
| 135 | } | ||
| 136 | |||
| 137 | async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) { | ||
| 138 | // We created a new account: fetch the playlists | ||
| 139 | if (created === true && actor.Account && accountPlaylistsUrl) { | ||
| 140 | const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' } | ||
| 141 | await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | ||
| 142 | } | ||
| 143 | } | ||
diff --git a/server/lib/activitypub/actors/image.ts b/server/lib/activitypub/actors/image.ts deleted file mode 100644 index e1d29af5b..000000000 --- a/server/lib/activitypub/actors/image.ts +++ /dev/null | |||
| @@ -1,112 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize/types' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { ActorImageModel } from '@server/models/actor/actor-image' | ||
| 4 | import { MActorImage, MActorImages } from '@server/types/models' | ||
| 5 | import { ActorImageType } from '@shared/models' | ||
| 6 | |||
| 7 | type ImageInfo = { | ||
| 8 | name: string | ||
| 9 | fileUrl: string | ||
| 10 | height: number | ||
| 11 | width: number | ||
| 12 | onDisk?: boolean | ||
| 13 | } | ||
| 14 | |||
| 15 | async function updateActorImages (actor: MActorImages, type: ActorImageType, imagesInfo: ImageInfo[], t: Transaction) { | ||
| 16 | const getAvatarsOrBanners = () => { | ||
| 17 | const result = type === ActorImageType.AVATAR | ||
| 18 | ? actor.Avatars | ||
| 19 | : actor.Banners | ||
| 20 | |||
| 21 | return result || [] | ||
| 22 | } | ||
| 23 | |||
| 24 | if (imagesInfo.length === 0) { | ||
| 25 | await deleteActorImages(actor, type, t) | ||
| 26 | } | ||
| 27 | |||
| 28 | // Cleanup old images that did not have a width | ||
| 29 | for (const oldImageModel of getAvatarsOrBanners()) { | ||
| 30 | if (oldImageModel.width) continue | ||
| 31 | |||
| 32 | await safeDeleteActorImage(actor, oldImageModel, type, t) | ||
| 33 | } | ||
| 34 | |||
| 35 | for (const imageInfo of imagesInfo) { | ||
| 36 | const oldImageModel = getAvatarsOrBanners().find(i => imageInfo.width && i.width === imageInfo.width) | ||
| 37 | |||
| 38 | if (oldImageModel) { | ||
| 39 | // Don't update the avatar if the file URL did not change | ||
| 40 | if (imageInfo.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) { | ||
| 41 | continue | ||
| 42 | } | ||
| 43 | |||
| 44 | await safeDeleteActorImage(actor, oldImageModel, type, t) | ||
| 45 | } | ||
| 46 | |||
| 47 | const imageModel = await ActorImageModel.create({ | ||
| 48 | filename: imageInfo.name, | ||
| 49 | onDisk: imageInfo.onDisk ?? false, | ||
| 50 | fileUrl: imageInfo.fileUrl, | ||
| 51 | height: imageInfo.height, | ||
| 52 | width: imageInfo.width, | ||
| 53 | type, | ||
| 54 | actorId: actor.id | ||
| 55 | }, { transaction: t }) | ||
| 56 | |||
| 57 | addActorImage(actor, type, imageModel) | ||
| 58 | } | ||
| 59 | |||
| 60 | return actor | ||
| 61 | } | ||
| 62 | |||
| 63 | async function deleteActorImages (actor: MActorImages, type: ActorImageType, t: Transaction) { | ||
| 64 | try { | ||
| 65 | const association = buildAssociationName(type) | ||
| 66 | |||
| 67 | for (const image of actor[association]) { | ||
| 68 | await image.destroy({ transaction: t }) | ||
| 69 | } | ||
| 70 | |||
| 71 | actor[association] = [] | ||
| 72 | } catch (err) { | ||
| 73 | logger.error('Cannot remove old image of actor %s.', actor.url, { err }) | ||
| 74 | } | ||
| 75 | |||
| 76 | return actor | ||
| 77 | } | ||
| 78 | |||
| 79 | async function safeDeleteActorImage (actor: MActorImages, toDelete: MActorImage, type: ActorImageType, t: Transaction) { | ||
| 80 | try { | ||
| 81 | await toDelete.destroy({ transaction: t }) | ||
| 82 | |||
| 83 | const association = buildAssociationName(type) | ||
| 84 | actor[association] = actor[association].filter(image => image.id !== toDelete.id) | ||
| 85 | } catch (err) { | ||
| 86 | logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) | ||
| 87 | } | ||
| 88 | } | ||
| 89 | |||
| 90 | // --------------------------------------------------------------------------- | ||
| 91 | |||
| 92 | export { | ||
| 93 | ImageInfo, | ||
| 94 | |||
| 95 | updateActorImages, | ||
| 96 | deleteActorImages | ||
| 97 | } | ||
| 98 | |||
| 99 | // --------------------------------------------------------------------------- | ||
| 100 | |||
| 101 | function addActorImage (actor: MActorImages, type: ActorImageType, imageModel: MActorImage) { | ||
| 102 | const association = buildAssociationName(type) | ||
| 103 | if (!actor[association]) actor[association] = [] | ||
| 104 | |||
| 105 | actor[association].push(imageModel) | ||
| 106 | } | ||
| 107 | |||
| 108 | function buildAssociationName (type: ActorImageType) { | ||
| 109 | return type === ActorImageType.AVATAR | ||
| 110 | ? 'Avatars' | ||
| 111 | : 'Banners' | ||
| 112 | } | ||
diff --git a/server/lib/activitypub/actors/index.ts b/server/lib/activitypub/actors/index.ts deleted file mode 100644 index 5ee2a6f1a..000000000 --- a/server/lib/activitypub/actors/index.ts +++ /dev/null | |||
| @@ -1,6 +0,0 @@ | |||
| 1 | export * from './get' | ||
| 2 | export * from './image' | ||
| 3 | export * from './keys' | ||
| 4 | export * from './refresh' | ||
| 5 | export * from './updater' | ||
| 6 | export * from './webfinger' | ||
diff --git a/server/lib/activitypub/actors/keys.ts b/server/lib/activitypub/actors/keys.ts deleted file mode 100644 index c3d18abd8..000000000 --- a/server/lib/activitypub/actors/keys.ts +++ /dev/null | |||
| @@ -1,16 +0,0 @@ | |||
| 1 | import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto' | ||
| 2 | import { MActor } from '@server/types/models' | ||
| 3 | |||
| 4 | // Set account keys, this could be long so process after the account creation and do not block the client | ||
| 5 | async function generateAndSaveActorKeys <T extends MActor> (actor: T) { | ||
| 6 | const { publicKey, privateKey } = await createPrivateAndPublicKeys() | ||
| 7 | |||
| 8 | actor.publicKey = publicKey | ||
| 9 | actor.privateKey = privateKey | ||
| 10 | |||
| 11 | return actor.save() | ||
| 12 | } | ||
| 13 | |||
| 14 | export { | ||
| 15 | generateAndSaveActorKeys | ||
| 16 | } | ||
diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts deleted file mode 100644 index d15cb5e90..000000000 --- a/server/lib/activitypub/actors/refresh.ts +++ /dev/null | |||
| @@ -1,81 +0,0 @@ | |||
| 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 2 | import { CachePromiseFactory } from '@server/helpers/promise-cache' | ||
| 3 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
| 4 | import { ActorLoadByUrlType } from '@server/lib/model-loaders' | ||
| 5 | import { ActorModel } from '@server/models/actor/actor' | ||
| 6 | import { MActorAccountChannelId, MActorFull } from '@server/types/models' | ||
| 7 | import { HttpStatusCode } from '@shared/models' | ||
| 8 | import { fetchRemoteActor } from './shared' | ||
| 9 | import { APActorUpdater } from './updater' | ||
| 10 | import { getUrlFromWebfinger } from './webfinger' | ||
| 11 | |||
| 12 | type RefreshResult <T> = Promise<{ actor: T | MActorFull, refreshed: boolean }> | ||
| 13 | |||
| 14 | type RefreshOptions <T> = { | ||
| 15 | actor: T | ||
| 16 | fetchedType: ActorLoadByUrlType | ||
| 17 | } | ||
| 18 | |||
| 19 | const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url) | ||
| 20 | |||
| 21 | function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <T> { | ||
| 22 | const actorArg = options.actor | ||
| 23 | if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false }) | ||
| 24 | |||
| 25 | return promiseCache.run(options) | ||
| 26 | } | ||
| 27 | |||
| 28 | export { | ||
| 29 | refreshActorIfNeeded | ||
| 30 | } | ||
| 31 | |||
| 32 | // --------------------------------------------------------------------------- | ||
| 33 | |||
| 34 | async function doRefresh <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <MActorFull> { | ||
| 35 | const { actor: actorArg, fetchedType } = options | ||
| 36 | |||
| 37 | // We need more attributes | ||
| 38 | const actor = fetchedType === 'all' | ||
| 39 | ? actorArg as MActorFull | ||
| 40 | : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
| 41 | |||
| 42 | const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url) | ||
| 43 | |||
| 44 | logger.info('Refreshing actor %s.', actor.url, lTags()) | ||
| 45 | |||
| 46 | try { | ||
| 47 | const actorUrl = await getActorUrl(actor) | ||
| 48 | const { actorObject } = await fetchRemoteActor(actorUrl) | ||
| 49 | |||
| 50 | if (actorObject === undefined) { | ||
| 51 | logger.info('Cannot fetch remote actor %s in refresh actor.', actorUrl) | ||
| 52 | return { actor, refreshed: false } | ||
| 53 | } | ||
| 54 | |||
| 55 | const updater = new APActorUpdater(actorObject, actor) | ||
| 56 | await updater.update() | ||
| 57 | |||
| 58 | return { refreshed: true, actor } | ||
| 59 | } catch (err) { | ||
| 60 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
| 61 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url, lTags()) | ||
| 62 | |||
| 63 | actor.Account | ||
| 64 | ? await actor.Account.destroy() | ||
| 65 | : await actor.VideoChannel.destroy() | ||
| 66 | |||
| 67 | return { actor: undefined, refreshed: false } | ||
| 68 | } | ||
| 69 | |||
| 70 | logger.info('Cannot refresh actor %s.', actor.url, { err, ...lTags() }) | ||
| 71 | return { actor, refreshed: false } | ||
| 72 | } | ||
| 73 | } | ||
| 74 | |||
| 75 | function getActorUrl (actor: MActorFull) { | ||
| 76 | return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
| 77 | .catch(err => { | ||
| 78 | logger.warn('Cannot get actor URL from webfinger, keeping the old one.', { err }) | ||
| 79 | return actor.url | ||
| 80 | }) | ||
| 81 | } | ||
diff --git a/server/lib/activitypub/actors/shared/creator.ts b/server/lib/activitypub/actors/shared/creator.ts deleted file mode 100644 index 500bc9912..000000000 --- a/server/lib/activitypub/actors/shared/creator.ts +++ /dev/null | |||
| @@ -1,149 +0,0 @@ | |||
| 1 | import { Op, Transaction } from 'sequelize' | ||
| 2 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 3 | import { AccountModel } from '@server/models/account/account' | ||
| 4 | import { ActorModel } from '@server/models/actor/actor' | ||
| 5 | import { ServerModel } from '@server/models/server/server' | ||
| 6 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
| 7 | import { MAccount, MAccountDefault, MActor, MActorFullActor, MActorId, MActorImages, MChannel, MServer } from '@server/types/models' | ||
| 8 | import { ActivityPubActor, ActorImageType } from '@shared/models' | ||
| 9 | import { updateActorImages } from '../image' | ||
| 10 | import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImagesInfoFromObject } from './object-to-model-attributes' | ||
| 11 | import { fetchActorFollowsCount } from './url-to-object' | ||
| 12 | |||
| 13 | export class APActorCreator { | ||
| 14 | |||
| 15 | constructor ( | ||
| 16 | private readonly actorObject: ActivityPubActor, | ||
| 17 | private readonly ownerActor?: MActorFullActor | ||
| 18 | ) { | ||
| 19 | |||
| 20 | } | ||
| 21 | |||
| 22 | async create (): Promise<MActorFullActor> { | ||
| 23 | const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject) | ||
| 24 | |||
| 25 | const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount)) | ||
| 26 | |||
| 27 | return sequelizeTypescript.transaction(async t => { | ||
| 28 | const server = await this.setServer(actorInstance, t) | ||
| 29 | |||
| 30 | const { actorCreated, created } = await this.saveActor(actorInstance, t) | ||
| 31 | |||
| 32 | await this.setImageIfNeeded(actorCreated, ActorImageType.AVATAR, t) | ||
| 33 | await this.setImageIfNeeded(actorCreated, ActorImageType.BANNER, t) | ||
| 34 | |||
| 35 | await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t) | ||
| 36 | |||
| 37 | if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance | ||
| 38 | actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault | ||
| 39 | actorCreated.Account.Actor = actorCreated | ||
| 40 | } | ||
| 41 | |||
| 42 | if (actorCreated.type === 'Group') { // Video channel | ||
| 43 | const channel = await this.saveVideoChannel(actorCreated, t) | ||
| 44 | actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account }) | ||
| 45 | } | ||
| 46 | |||
| 47 | actorCreated.Server = server | ||
| 48 | |||
| 49 | return actorCreated | ||
| 50 | }) | ||
| 51 | } | ||
| 52 | |||
| 53 | private async setServer (actor: MActor, t: Transaction) { | ||
| 54 | const actorHost = new URL(actor.url).host | ||
| 55 | |||
| 56 | const serverOptions = { | ||
| 57 | where: { | ||
| 58 | host: actorHost | ||
| 59 | }, | ||
| 60 | defaults: { | ||
| 61 | host: actorHost | ||
| 62 | }, | ||
| 63 | transaction: t | ||
| 64 | } | ||
| 65 | const [ server ] = await ServerModel.findOrCreate(serverOptions) | ||
| 66 | |||
| 67 | // Save our new account in database | ||
| 68 | actor.serverId = server.id | ||
| 69 | |||
| 70 | return server as MServer | ||
| 71 | } | ||
| 72 | |||
| 73 | private async setImageIfNeeded (actor: MActor, type: ActorImageType, t: Transaction) { | ||
| 74 | const imagesInfo = getImagesInfoFromObject(this.actorObject, type) | ||
| 75 | if (imagesInfo.length === 0) return | ||
| 76 | |||
| 77 | return updateActorImages(actor as MActorImages, type, imagesInfo, t) | ||
| 78 | } | ||
| 79 | |||
| 80 | private async saveActor (actor: MActor, t: Transaction) { | ||
| 81 | // Force the actor creation using findOrCreate() instead of save() | ||
| 82 | // Sometimes Sequelize skips the save() when it thinks the instance already exists | ||
| 83 | // (which could be false in a retried query) | ||
| 84 | const [ actorCreated, created ] = await ActorModel.findOrCreate<MActorFullActor>({ | ||
| 85 | defaults: actor.toJSON(), | ||
| 86 | where: { | ||
| 87 | [Op.or]: [ | ||
| 88 | { | ||
| 89 | url: actor.url | ||
| 90 | }, | ||
| 91 | { | ||
| 92 | serverId: actor.serverId, | ||
| 93 | preferredUsername: actor.preferredUsername | ||
| 94 | } | ||
| 95 | ] | ||
| 96 | }, | ||
| 97 | transaction: t | ||
| 98 | }) | ||
| 99 | |||
| 100 | return { actorCreated, created } | ||
| 101 | } | ||
| 102 | |||
| 103 | private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) { | ||
| 104 | // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards | ||
| 105 | if (created !== true && actorCreated.url !== newActor.url) { | ||
| 106 | // Only fix http://example.com/account/djidane to https://example.com/account/djidane | ||
| 107 | if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) { | ||
| 108 | throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`) | ||
| 109 | } | ||
| 110 | |||
| 111 | actorCreated.url = newActor.url | ||
| 112 | await actorCreated.save({ transaction: t }) | ||
| 113 | } | ||
| 114 | } | ||
| 115 | |||
| 116 | private async saveAccount (actor: MActorId, t: Transaction) { | ||
| 117 | const [ accountCreated ] = await AccountModel.findOrCreate({ | ||
| 118 | defaults: { | ||
| 119 | name: getActorDisplayNameFromObject(this.actorObject), | ||
| 120 | description: this.actorObject.summary, | ||
| 121 | actorId: actor.id | ||
| 122 | }, | ||
| 123 | where: { | ||
| 124 | actorId: actor.id | ||
| 125 | }, | ||
| 126 | transaction: t | ||
| 127 | }) | ||
| 128 | |||
| 129 | return accountCreated as MAccount | ||
| 130 | } | ||
| 131 | |||
| 132 | private async saveVideoChannel (actor: MActorId, t: Transaction) { | ||
| 133 | const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ | ||
| 134 | defaults: { | ||
| 135 | name: getActorDisplayNameFromObject(this.actorObject), | ||
| 136 | description: this.actorObject.summary, | ||
| 137 | support: this.actorObject.support, | ||
| 138 | actorId: actor.id, | ||
| 139 | accountId: this.ownerActor.Account.id | ||
| 140 | }, | ||
| 141 | where: { | ||
| 142 | actorId: actor.id | ||
| 143 | }, | ||
| 144 | transaction: t | ||
| 145 | }) | ||
| 146 | |||
| 147 | return videoChannelCreated as MChannel | ||
| 148 | } | ||
| 149 | } | ||
diff --git a/server/lib/activitypub/actors/shared/index.ts b/server/lib/activitypub/actors/shared/index.ts deleted file mode 100644 index 52af1a8e1..000000000 --- a/server/lib/activitypub/actors/shared/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './creator' | ||
| 2 | export * from './object-to-model-attributes' | ||
| 3 | export * from './url-to-object' | ||
diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts deleted file mode 100644 index 3ce332681..000000000 --- a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts +++ /dev/null | |||
| @@ -1,84 +0,0 @@ | |||
| 1 | import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' | ||
| 2 | import { MIMETYPES } from '@server/initializers/constants' | ||
| 3 | import { ActorModel } from '@server/models/actor/actor' | ||
| 4 | import { FilteredModelAttributes } from '@server/types' | ||
| 5 | import { getLowercaseExtension } from '@shared/core-utils' | ||
| 6 | import { buildUUID } from '@shared/extra-utils' | ||
| 7 | import { ActivityIconObject, ActivityPubActor, ActorImageType } from '@shared/models' | ||
| 8 | |||
| 9 | function getActorAttributesFromObject ( | ||
| 10 | actorObject: ActivityPubActor, | ||
| 11 | followersCount: number, | ||
| 12 | followingCount: number | ||
| 13 | ): FilteredModelAttributes<ActorModel> { | ||
| 14 | return { | ||
| 15 | type: actorObject.type, | ||
| 16 | preferredUsername: actorObject.preferredUsername, | ||
| 17 | url: actorObject.id, | ||
| 18 | publicKey: actorObject.publicKey.publicKeyPem, | ||
| 19 | privateKey: null, | ||
| 20 | followersCount, | ||
| 21 | followingCount, | ||
| 22 | inboxUrl: actorObject.inbox, | ||
| 23 | outboxUrl: actorObject.outbox, | ||
| 24 | followersUrl: actorObject.followers, | ||
| 25 | followingUrl: actorObject.following, | ||
| 26 | |||
| 27 | sharedInboxUrl: actorObject.endpoints?.sharedInbox | ||
| 28 | ? actorObject.endpoints.sharedInbox | ||
| 29 | : null | ||
| 30 | } | ||
| 31 | } | ||
| 32 | |||
| 33 | function getImagesInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType) { | ||
| 34 | const iconsOrImages = type === ActorImageType.AVATAR | ||
| 35 | ? actorObject.icon | ||
| 36 | : actorObject.image | ||
| 37 | |||
| 38 | return normalizeIconOrImage(iconsOrImages) | ||
| 39 | .map(iconOrImage => { | ||
| 40 | const mimetypes = MIMETYPES.IMAGE | ||
| 41 | |||
| 42 | if (iconOrImage.type !== 'Image' || !isActivityPubUrlValid(iconOrImage.url)) return undefined | ||
| 43 | |||
| 44 | let extension: string | ||
| 45 | |||
| 46 | if (iconOrImage.mediaType) { | ||
| 47 | extension = mimetypes.MIMETYPE_EXT[iconOrImage.mediaType] | ||
| 48 | } else { | ||
| 49 | const tmp = getLowercaseExtension(iconOrImage.url) | ||
| 50 | |||
| 51 | if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp | ||
| 52 | } | ||
| 53 | |||
| 54 | if (!extension) return undefined | ||
| 55 | |||
| 56 | return { | ||
| 57 | name: buildUUID() + extension, | ||
| 58 | fileUrl: iconOrImage.url, | ||
| 59 | height: iconOrImage.height, | ||
| 60 | width: iconOrImage.width, | ||
| 61 | type | ||
| 62 | } | ||
| 63 | }) | ||
| 64 | .filter(i => !!i) | ||
| 65 | } | ||
| 66 | |||
| 67 | function getActorDisplayNameFromObject (actorObject: ActivityPubActor) { | ||
| 68 | return actorObject.name || actorObject.preferredUsername | ||
| 69 | } | ||
| 70 | |||
| 71 | export { | ||
| 72 | getActorAttributesFromObject, | ||
| 73 | getImagesInfoFromObject, | ||
| 74 | getActorDisplayNameFromObject | ||
| 75 | } | ||
| 76 | |||
| 77 | // --------------------------------------------------------------------------- | ||
| 78 | |||
| 79 | function normalizeIconOrImage (icon: ActivityIconObject | ActivityIconObject[]): ActivityIconObject[] { | ||
| 80 | if (Array.isArray(icon)) return icon | ||
| 81 | if (icon) return [ icon ] | ||
| 82 | |||
| 83 | return [] | ||
| 84 | } | ||
diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts deleted file mode 100644 index 73766bd50..000000000 --- a/server/lib/activitypub/actors/shared/url-to-object.ts +++ /dev/null | |||
| @@ -1,56 +0,0 @@ | |||
| 1 | import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models' | ||
| 4 | import { fetchAP } from '../../activity' | ||
| 5 | import { checkUrlsSameHost } from '../../url' | ||
| 6 | |||
| 7 | async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> { | ||
| 8 | logger.info('Fetching remote actor %s.', actorUrl) | ||
| 9 | |||
| 10 | const { body, statusCode } = await fetchAP<ActivityPubActor>(actorUrl) | ||
| 11 | |||
| 12 | if (sanitizeAndCheckActorObject(body) === false) { | ||
| 13 | logger.debug('Remote actor JSON is not valid.', { actorJSON: body }) | ||
| 14 | return { actorObject: undefined, statusCode } | ||
| 15 | } | ||
| 16 | |||
| 17 | if (checkUrlsSameHost(body.id, actorUrl) !== true) { | ||
| 18 | logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id) | ||
| 19 | return { actorObject: undefined, statusCode } | ||
| 20 | } | ||
| 21 | |||
| 22 | return { | ||
| 23 | statusCode, | ||
| 24 | |||
| 25 | actorObject: body | ||
| 26 | } | ||
| 27 | } | ||
| 28 | |||
| 29 | async function fetchActorFollowsCount (actorObject: ActivityPubActor) { | ||
| 30 | let followersCount = 0 | ||
| 31 | let followingCount = 0 | ||
| 32 | |||
| 33 | if (actorObject.followers) followersCount = await fetchActorTotalItems(actorObject.followers) | ||
| 34 | if (actorObject.following) followingCount = await fetchActorTotalItems(actorObject.following) | ||
| 35 | |||
| 36 | return { followersCount, followingCount } | ||
| 37 | } | ||
| 38 | |||
| 39 | // --------------------------------------------------------------------------- | ||
| 40 | export { | ||
| 41 | fetchActorFollowsCount, | ||
| 42 | fetchRemoteActor | ||
| 43 | } | ||
| 44 | |||
| 45 | // --------------------------------------------------------------------------- | ||
| 46 | |||
| 47 | async function fetchActorTotalItems (url: string) { | ||
| 48 | try { | ||
| 49 | const { body } = await fetchAP<ActivityPubOrderedCollection<unknown>>(url) | ||
| 50 | |||
| 51 | return body.totalItems || 0 | ||
| 52 | } catch (err) { | ||
| 53 | logger.info('Cannot fetch remote actor count %s.', url, { err }) | ||
| 54 | return 0 | ||
| 55 | } | ||
| 56 | } | ||
diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts deleted file mode 100644 index 5a92e7a22..000000000 --- a/server/lib/activitypub/actors/updater.ts +++ /dev/null | |||
| @@ -1,91 +0,0 @@ | |||
| 1 | import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { AccountModel } from '@server/models/account/account' | ||
| 4 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
| 5 | import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' | ||
| 6 | import { ActivityPubActor, ActorImageType } from '@shared/models' | ||
| 7 | import { getOrCreateAPOwner } from './get' | ||
| 8 | import { updateActorImages } from './image' | ||
| 9 | import { fetchActorFollowsCount } from './shared' | ||
| 10 | import { getImagesInfoFromObject } from './shared/object-to-model-attributes' | ||
| 11 | |||
| 12 | export class APActorUpdater { | ||
| 13 | |||
| 14 | private readonly accountOrChannel: MAccount | MChannel | ||
| 15 | |||
| 16 | constructor ( | ||
| 17 | private readonly actorObject: ActivityPubActor, | ||
| 18 | private readonly actor: MActorFull | ||
| 19 | ) { | ||
| 20 | if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel | ||
| 21 | else this.accountOrChannel = this.actor.Account | ||
| 22 | } | ||
| 23 | |||
| 24 | async update () { | ||
| 25 | const avatarsInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.AVATAR) | ||
| 26 | const bannersInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.BANNER) | ||
| 27 | |||
| 28 | try { | ||
| 29 | await this.updateActorInstance(this.actor, this.actorObject) | ||
| 30 | |||
| 31 | this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername | ||
| 32 | this.accountOrChannel.description = this.actorObject.summary | ||
| 33 | |||
| 34 | if (this.accountOrChannel instanceof VideoChannelModel) { | ||
| 35 | const owner = await getOrCreateAPOwner(this.actorObject, this.actorObject.url) | ||
| 36 | this.accountOrChannel.accountId = owner.Account.id | ||
| 37 | this.accountOrChannel.Account = owner.Account as AccountModel | ||
| 38 | |||
| 39 | this.accountOrChannel.support = this.actorObject.support | ||
| 40 | } | ||
| 41 | |||
| 42 | await runInReadCommittedTransaction(async t => { | ||
| 43 | await updateActorImages(this.actor, ActorImageType.BANNER, bannersInfo, t) | ||
| 44 | await updateActorImages(this.actor, ActorImageType.AVATAR, avatarsInfo, t) | ||
| 45 | }) | ||
| 46 | |||
| 47 | await runInReadCommittedTransaction(async t => { | ||
| 48 | await this.actor.save({ transaction: t }) | ||
| 49 | await this.accountOrChannel.save({ transaction: t }) | ||
| 50 | }) | ||
| 51 | |||
| 52 | logger.info('Remote account %s updated', this.actorObject.url) | ||
| 53 | } catch (err) { | ||
| 54 | if (this.actor !== undefined) { | ||
| 55 | await resetSequelizeInstance(this.actor) | ||
| 56 | } | ||
| 57 | |||
| 58 | if (this.accountOrChannel !== undefined) { | ||
| 59 | await resetSequelizeInstance(this.accountOrChannel) | ||
| 60 | } | ||
| 61 | |||
| 62 | // This is just a debug because we will retry the insert | ||
| 63 | logger.debug('Cannot update the remote account.', { err }) | ||
| 64 | throw err | ||
| 65 | } | ||
| 66 | } | ||
| 67 | |||
| 68 | private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) { | ||
| 69 | const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject) | ||
| 70 | |||
| 71 | actorInstance.type = actorObject.type | ||
| 72 | actorInstance.preferredUsername = actorObject.preferredUsername | ||
| 73 | actorInstance.url = actorObject.id | ||
| 74 | actorInstance.publicKey = actorObject.publicKey.publicKeyPem | ||
| 75 | actorInstance.followersCount = followersCount | ||
| 76 | actorInstance.followingCount = followingCount | ||
| 77 | actorInstance.inboxUrl = actorObject.inbox | ||
| 78 | actorInstance.outboxUrl = actorObject.outbox | ||
| 79 | actorInstance.followersUrl = actorObject.followers | ||
| 80 | actorInstance.followingUrl = actorObject.following | ||
| 81 | |||
| 82 | if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published) | ||
| 83 | |||
| 84 | if (actorObject.endpoints?.sharedInbox) { | ||
| 85 | actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox | ||
| 86 | } | ||
| 87 | |||
| 88 | // Force actor update | ||
| 89 | actorInstance.changed('updatedAt', true) | ||
| 90 | } | ||
| 91 | } | ||
diff --git a/server/lib/activitypub/actors/webfinger.ts b/server/lib/activitypub/actors/webfinger.ts deleted file mode 100644 index b20a724da..000000000 --- a/server/lib/activitypub/actors/webfinger.ts +++ /dev/null | |||
| @@ -1,67 +0,0 @@ | |||
| 1 | import WebFinger from 'webfinger.js' | ||
| 2 | import { isProdInstance } from '@server/helpers/core-utils' | ||
| 3 | import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' | ||
| 4 | import { REQUEST_TIMEOUTS, WEBSERVER } from '@server/initializers/constants' | ||
| 5 | import { ActorModel } from '@server/models/actor/actor' | ||
| 6 | import { MActorFull } from '@server/types/models' | ||
| 7 | import { WebFingerData } from '@shared/models' | ||
| 8 | |||
| 9 | const webfinger = new WebFinger({ | ||
| 10 | webfist_fallback: false, | ||
| 11 | tls_only: isProdInstance(), | ||
| 12 | uri_fallback: false, | ||
| 13 | request_timeout: REQUEST_TIMEOUTS.DEFAULT | ||
| 14 | }) | ||
| 15 | |||
| 16 | async function loadActorUrlOrGetFromWebfinger (uriArg: string) { | ||
| 17 | // Handle strings like @toto@example.com | ||
| 18 | const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg | ||
| 19 | |||
| 20 | const [ name, host ] = uri.split('@') | ||
| 21 | let actor: MActorFull | ||
| 22 | |||
| 23 | if (!host || host === WEBSERVER.HOST) { | ||
| 24 | actor = await ActorModel.loadLocalByName(name) | ||
| 25 | } else { | ||
| 26 | actor = await ActorModel.loadByNameAndHost(name, host) | ||
| 27 | } | ||
| 28 | |||
| 29 | if (actor) return actor.url | ||
| 30 | |||
| 31 | return getUrlFromWebfinger(uri) | ||
| 32 | } | ||
| 33 | |||
| 34 | async function getUrlFromWebfinger (uri: string) { | ||
| 35 | const webfingerData: WebFingerData = await webfingerLookup(uri) | ||
| 36 | return getLinkOrThrow(webfingerData) | ||
| 37 | } | ||
| 38 | |||
| 39 | // --------------------------------------------------------------------------- | ||
| 40 | |||
| 41 | export { | ||
| 42 | getUrlFromWebfinger, | ||
| 43 | loadActorUrlOrGetFromWebfinger | ||
| 44 | } | ||
| 45 | |||
| 46 | // --------------------------------------------------------------------------- | ||
| 47 | |||
| 48 | function getLinkOrThrow (webfingerData: WebFingerData) { | ||
| 49 | if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.') | ||
| 50 | |||
| 51 | const selfLink = webfingerData.links.find(l => l.rel === 'self') | ||
| 52 | if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) { | ||
| 53 | throw new Error('Cannot find self link or href is not a valid URL.') | ||
| 54 | } | ||
| 55 | |||
| 56 | return selfLink.href | ||
| 57 | } | ||
| 58 | |||
| 59 | function webfingerLookup (nameWithHost: string) { | ||
| 60 | return new Promise<WebFingerData>((res, rej) => { | ||
| 61 | webfinger.lookup(nameWithHost, (err, p) => { | ||
| 62 | if (err) return rej(err) | ||
| 63 | |||
| 64 | return res(p.object) | ||
| 65 | }) | ||
| 66 | }) | ||
| 67 | } | ||
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts deleted file mode 100644 index 6f5491387..000000000 --- a/server/lib/activitypub/audience.ts +++ /dev/null | |||
| @@ -1,34 +0,0 @@ | |||
| 1 | import { ActivityAudience } from '../../../shared/models/activitypub' | ||
| 2 | import { ACTIVITY_PUB } from '../../initializers/constants' | ||
| 3 | import { MActorFollowersUrl } from '../../types/models' | ||
| 4 | |||
| 5 | function getAudience (actorSender: MActorFollowersUrl, isPublic = true) { | ||
| 6 | return buildAudience([ actorSender.followersUrl ], isPublic) | ||
| 7 | } | ||
| 8 | |||
| 9 | function buildAudience (followerUrls: string[], isPublic = true) { | ||
| 10 | let to: string[] = [] | ||
| 11 | let cc: string[] = [] | ||
| 12 | |||
| 13 | if (isPublic) { | ||
| 14 | to = [ ACTIVITY_PUB.PUBLIC ] | ||
| 15 | cc = followerUrls | ||
| 16 | } else { // Unlisted | ||
| 17 | to = [] | ||
| 18 | cc = [] | ||
| 19 | } | ||
| 20 | |||
| 21 | return { to, cc } | ||
| 22 | } | ||
| 23 | |||
| 24 | function audiencify<T> (object: T, audience: ActivityAudience) { | ||
| 25 | return { ...audience, ...object } | ||
| 26 | } | ||
| 27 | |||
| 28 | // --------------------------------------------------------------------------- | ||
| 29 | |||
| 30 | export { | ||
| 31 | buildAudience, | ||
| 32 | getAudience, | ||
| 33 | audiencify | ||
| 34 | } | ||
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts deleted file mode 100644 index c3acd7112..000000000 --- a/server/lib/activitypub/cache-file.ts +++ /dev/null | |||
| @@ -1,82 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' | ||
| 3 | import { CacheFileObject, VideoStreamingPlaylistType } from '@shared/models' | ||
| 4 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | ||
| 5 | |||
| 6 | async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { | ||
| 7 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) | ||
| 8 | |||
| 9 | if (redundancyModel) { | ||
| 10 | return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) | ||
| 11 | } | ||
| 12 | |||
| 13 | return createCacheFile(cacheFileObject, video, byActor, t) | ||
| 14 | } | ||
| 15 | |||
| 16 | // --------------------------------------------------------------------------- | ||
| 17 | |||
| 18 | export { | ||
| 19 | createOrUpdateCacheFile | ||
| 20 | } | ||
| 21 | |||
| 22 | // --------------------------------------------------------------------------- | ||
| 23 | |||
| 24 | function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { | ||
| 25 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) | ||
| 26 | |||
| 27 | return VideoRedundancyModel.create(attributes, { transaction: t }) | ||
| 28 | } | ||
| 29 | |||
| 30 | function updateCacheFile ( | ||
| 31 | cacheFileObject: CacheFileObject, | ||
| 32 | redundancyModel: MVideoRedundancy, | ||
| 33 | video: MVideoWithAllFiles, | ||
| 34 | byActor: MActorId, | ||
| 35 | t: Transaction | ||
| 36 | ) { | ||
| 37 | if (redundancyModel.actorId !== byActor.id) { | ||
| 38 | throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') | ||
| 39 | } | ||
| 40 | |||
| 41 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) | ||
| 42 | |||
| 43 | redundancyModel.expiresOn = attributes.expiresOn | ||
| 44 | redundancyModel.fileUrl = attributes.fileUrl | ||
| 45 | |||
| 46 | return redundancyModel.save({ transaction: t }) | ||
| 47 | } | ||
| 48 | |||
| 49 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { | ||
| 50 | |||
| 51 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { | ||
| 52 | const url = cacheFileObject.url | ||
| 53 | |||
| 54 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
| 55 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
| 56 | |||
| 57 | return { | ||
| 58 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, | ||
| 59 | url: cacheFileObject.id, | ||
| 60 | fileUrl: url.href, | ||
| 61 | strategy: null, | ||
| 62 | videoStreamingPlaylistId: playlist.id, | ||
| 63 | actorId: byActor.id | ||
| 64 | } | ||
| 65 | } | ||
| 66 | |||
| 67 | const url = cacheFileObject.url | ||
| 68 | const videoFile = video.VideoFiles.find(f => { | ||
| 69 | return f.resolution === url.height && f.fps === url.fps | ||
| 70 | }) | ||
| 71 | |||
| 72 | if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) | ||
| 73 | |||
| 74 | return { | ||
| 75 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, | ||
| 76 | url: cacheFileObject.id, | ||
| 77 | fileUrl: url.href, | ||
| 78 | strategy: null, | ||
| 79 | videoFileId: videoFile.id, | ||
| 80 | actorId: byActor.id | ||
| 81 | } | ||
| 82 | } | ||
diff --git a/server/lib/activitypub/collection.ts b/server/lib/activitypub/collection.ts deleted file mode 100644 index a176cab51..000000000 --- a/server/lib/activitypub/collection.ts +++ /dev/null | |||
| @@ -1,63 +0,0 @@ | |||
| 1 | import Bluebird from 'bluebird' | ||
| 2 | import validator from 'validator' | ||
| 3 | import { pageToStartAndCount } from '@server/helpers/core-utils' | ||
| 4 | import { ACTIVITY_PUB } from '@server/initializers/constants' | ||
| 5 | import { ResultList } from '@shared/models' | ||
| 6 | import { forceNumber } from '@shared/core-utils' | ||
| 7 | |||
| 8 | type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>> | ||
| 9 | |||
| 10 | async function activityPubCollectionPagination ( | ||
| 11 | baseUrl: string, | ||
| 12 | handler: ActivityPubCollectionPaginationHandler, | ||
| 13 | page?: any, | ||
| 14 | size = ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE | ||
| 15 | ) { | ||
| 16 | if (!page || !validator.isInt(page)) { | ||
| 17 | // We just display the first page URL, we only need the total items | ||
| 18 | const result = await handler(0, 1) | ||
| 19 | |||
| 20 | return { | ||
| 21 | id: baseUrl, | ||
| 22 | type: 'OrderedCollection', | ||
| 23 | totalItems: result.total, | ||
| 24 | first: result.data.length === 0 | ||
| 25 | ? undefined | ||
| 26 | : baseUrl + '?page=1' | ||
| 27 | } | ||
| 28 | } | ||
| 29 | |||
| 30 | const { start, count } = pageToStartAndCount(page, size) | ||
| 31 | const result = await handler(start, count) | ||
| 32 | |||
| 33 | let next: string | undefined | ||
| 34 | let prev: string | undefined | ||
| 35 | |||
| 36 | // Assert page is a number | ||
| 37 | page = forceNumber(page) | ||
| 38 | |||
| 39 | // There are more results | ||
| 40 | if (result.total > page * size) { | ||
| 41 | next = baseUrl + '?page=' + (page + 1) | ||
| 42 | } | ||
| 43 | |||
| 44 | if (page > 1) { | ||
| 45 | prev = baseUrl + '?page=' + (page - 1) | ||
| 46 | } | ||
| 47 | |||
| 48 | return { | ||
| 49 | id: baseUrl + '?page=' + page, | ||
| 50 | type: 'OrderedCollectionPage', | ||
| 51 | prev, | ||
| 52 | next, | ||
| 53 | partOf: baseUrl, | ||
| 54 | orderedItems: result.data, | ||
| 55 | totalItems: result.total | ||
| 56 | } | ||
| 57 | } | ||
| 58 | |||
| 59 | // --------------------------------------------------------------------------- | ||
| 60 | |||
| 61 | export { | ||
| 62 | activityPubCollectionPagination | ||
| 63 | } | ||
diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts deleted file mode 100644 index 87eb498a3..000000000 --- a/server/lib/activitypub/context.ts +++ /dev/null | |||
| @@ -1,212 +0,0 @@ | |||
| 1 | import { ContextType } from '@shared/models' | ||
| 2 | import { Hooks } from '../plugins/hooks' | ||
| 3 | |||
| 4 | async function activityPubContextify <T> (data: T, type: ContextType) { | ||
| 5 | return { ...await getContextData(type), ...data } | ||
| 6 | } | ||
| 7 | |||
| 8 | // --------------------------------------------------------------------------- | ||
| 9 | |||
| 10 | export { | ||
| 11 | getContextData, | ||
| 12 | activityPubContextify | ||
| 13 | } | ||
| 14 | |||
| 15 | // --------------------------------------------------------------------------- | ||
| 16 | |||
| 17 | type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) } | ||
| 18 | |||
| 19 | const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = { | ||
| 20 | Video: buildContext({ | ||
| 21 | Hashtag: 'as:Hashtag', | ||
| 22 | uuid: 'sc:identifier', | ||
| 23 | category: 'sc:category', | ||
| 24 | licence: 'sc:license', | ||
| 25 | subtitleLanguage: 'sc:subtitleLanguage', | ||
| 26 | sensitive: 'as:sensitive', | ||
| 27 | language: 'sc:inLanguage', | ||
| 28 | identifier: 'sc:identifier', | ||
| 29 | |||
| 30 | isLiveBroadcast: 'sc:isLiveBroadcast', | ||
| 31 | liveSaveReplay: { | ||
| 32 | '@type': 'sc:Boolean', | ||
| 33 | '@id': 'pt:liveSaveReplay' | ||
| 34 | }, | ||
| 35 | permanentLive: { | ||
| 36 | '@type': 'sc:Boolean', | ||
| 37 | '@id': 'pt:permanentLive' | ||
| 38 | }, | ||
| 39 | latencyMode: { | ||
| 40 | '@type': 'sc:Number', | ||
| 41 | '@id': 'pt:latencyMode' | ||
| 42 | }, | ||
| 43 | |||
| 44 | Infohash: 'pt:Infohash', | ||
| 45 | |||
| 46 | tileWidth: { | ||
| 47 | '@type': 'sc:Number', | ||
| 48 | '@id': 'pt:tileWidth' | ||
| 49 | }, | ||
| 50 | tileHeight: { | ||
| 51 | '@type': 'sc:Number', | ||
| 52 | '@id': 'pt:tileHeight' | ||
| 53 | }, | ||
| 54 | tileDuration: { | ||
| 55 | '@type': 'sc:Number', | ||
| 56 | '@id': 'pt:tileDuration' | ||
| 57 | }, | ||
| 58 | |||
| 59 | originallyPublishedAt: 'sc:datePublished', | ||
| 60 | |||
| 61 | uploadDate: 'sc:uploadDate', | ||
| 62 | |||
| 63 | views: { | ||
| 64 | '@type': 'sc:Number', | ||
| 65 | '@id': 'pt:views' | ||
| 66 | }, | ||
| 67 | state: { | ||
| 68 | '@type': 'sc:Number', | ||
| 69 | '@id': 'pt:state' | ||
| 70 | }, | ||
| 71 | size: { | ||
| 72 | '@type': 'sc:Number', | ||
| 73 | '@id': 'pt:size' | ||
| 74 | }, | ||
| 75 | fps: { | ||
| 76 | '@type': 'sc:Number', | ||
| 77 | '@id': 'pt:fps' | ||
| 78 | }, | ||
| 79 | commentsEnabled: { | ||
| 80 | '@type': 'sc:Boolean', | ||
| 81 | '@id': 'pt:commentsEnabled' | ||
| 82 | }, | ||
| 83 | downloadEnabled: { | ||
| 84 | '@type': 'sc:Boolean', | ||
| 85 | '@id': 'pt:downloadEnabled' | ||
| 86 | }, | ||
| 87 | waitTranscoding: { | ||
| 88 | '@type': 'sc:Boolean', | ||
| 89 | '@id': 'pt:waitTranscoding' | ||
| 90 | }, | ||
| 91 | support: { | ||
| 92 | '@type': 'sc:Text', | ||
| 93 | '@id': 'pt:support' | ||
| 94 | }, | ||
| 95 | likes: { | ||
| 96 | '@id': 'as:likes', | ||
| 97 | '@type': '@id' | ||
| 98 | }, | ||
| 99 | dislikes: { | ||
| 100 | '@id': 'as:dislikes', | ||
| 101 | '@type': '@id' | ||
| 102 | }, | ||
| 103 | shares: { | ||
| 104 | '@id': 'as:shares', | ||
| 105 | '@type': '@id' | ||
| 106 | }, | ||
| 107 | comments: { | ||
| 108 | '@id': 'as:comments', | ||
| 109 | '@type': '@id' | ||
| 110 | } | ||
| 111 | }), | ||
| 112 | |||
| 113 | Playlist: buildContext({ | ||
| 114 | Playlist: 'pt:Playlist', | ||
| 115 | PlaylistElement: 'pt:PlaylistElement', | ||
| 116 | position: { | ||
| 117 | '@type': 'sc:Number', | ||
| 118 | '@id': 'pt:position' | ||
| 119 | }, | ||
| 120 | startTimestamp: { | ||
| 121 | '@type': 'sc:Number', | ||
| 122 | '@id': 'pt:startTimestamp' | ||
| 123 | }, | ||
| 124 | stopTimestamp: { | ||
| 125 | '@type': 'sc:Number', | ||
| 126 | '@id': 'pt:stopTimestamp' | ||
| 127 | }, | ||
| 128 | uuid: 'sc:identifier' | ||
| 129 | }), | ||
| 130 | |||
| 131 | CacheFile: buildContext({ | ||
| 132 | expires: 'sc:expires', | ||
| 133 | CacheFile: 'pt:CacheFile' | ||
| 134 | }), | ||
| 135 | |||
| 136 | Flag: buildContext({ | ||
| 137 | Hashtag: 'as:Hashtag' | ||
| 138 | }), | ||
| 139 | |||
| 140 | Actor: buildContext({ | ||
| 141 | playlists: { | ||
| 142 | '@id': 'pt:playlists', | ||
| 143 | '@type': '@id' | ||
| 144 | }, | ||
| 145 | support: { | ||
| 146 | '@type': 'sc:Text', | ||
| 147 | '@id': 'pt:support' | ||
| 148 | }, | ||
| 149 | |||
| 150 | // TODO: remove in a few versions, introduced in 4.2 | ||
| 151 | icons: 'as:icon' | ||
| 152 | }), | ||
| 153 | |||
| 154 | WatchAction: buildContext({ | ||
| 155 | WatchAction: 'sc:WatchAction', | ||
| 156 | startTimestamp: { | ||
| 157 | '@type': 'sc:Number', | ||
| 158 | '@id': 'pt:startTimestamp' | ||
| 159 | }, | ||
| 160 | stopTimestamp: { | ||
| 161 | '@type': 'sc:Number', | ||
| 162 | '@id': 'pt:stopTimestamp' | ||
| 163 | }, | ||
| 164 | watchSection: { | ||
| 165 | '@type': 'sc:Number', | ||
| 166 | '@id': 'pt:stopTimestamp' | ||
| 167 | }, | ||
| 168 | uuid: 'sc:identifier' | ||
| 169 | }), | ||
| 170 | |||
| 171 | Collection: buildContext(), | ||
| 172 | Follow: buildContext(), | ||
| 173 | Reject: buildContext(), | ||
| 174 | Accept: buildContext(), | ||
| 175 | View: buildContext(), | ||
| 176 | Announce: buildContext(), | ||
| 177 | Comment: buildContext(), | ||
| 178 | Delete: buildContext(), | ||
| 179 | Rate: buildContext() | ||
| 180 | } | ||
| 181 | |||
| 182 | async function getContextData (type: ContextType) { | ||
| 183 | const contextData = await Hooks.wrapObject( | ||
| 184 | contextStore[type], | ||
| 185 | 'filter:activity-pub.activity.context.build.result' | ||
| 186 | ) | ||
| 187 | |||
| 188 | return { '@context': contextData } | ||
| 189 | } | ||
| 190 | |||
| 191 | function buildContext (contextValue?: ContextValue) { | ||
| 192 | const baseContext = [ | ||
| 193 | 'https://www.w3.org/ns/activitystreams', | ||
| 194 | 'https://w3id.org/security/v1', | ||
| 195 | { | ||
| 196 | RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' | ||
| 197 | } | ||
| 198 | ] | ||
| 199 | |||
| 200 | if (!contextValue) return baseContext | ||
| 201 | |||
| 202 | return [ | ||
| 203 | ...baseContext, | ||
| 204 | |||
| 205 | { | ||
| 206 | pt: 'https://joinpeertube.org/ns#', | ||
| 207 | sc: 'http://schema.org/', | ||
| 208 | |||
| 209 | ...contextValue | ||
| 210 | } | ||
| 211 | ] | ||
| 212 | } | ||
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts deleted file mode 100644 index b8348e8cf..000000000 --- a/server/lib/activitypub/crawl.ts +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | import Bluebird from 'bluebird' | ||
| 2 | import { URL } from 'url' | ||
| 3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 4 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | ||
| 5 | import { logger } from '../../helpers/logger' | ||
| 6 | import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants' | ||
| 7 | import { fetchAP } from './activity' | ||
| 8 | |||
| 9 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) | ||
| 10 | type CleanerFunction = (startedDate: Date) => Promise<any> | ||
| 11 | |||
| 12 | async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) { | ||
| 13 | let url = argUrl | ||
| 14 | |||
| 15 | logger.info('Crawling ActivityPub data on %s.', url) | ||
| 16 | |||
| 17 | const startDate = new Date() | ||
| 18 | |||
| 19 | const response = await fetchAP<ActivityPubOrderedCollection<T>>(url) | ||
| 20 | const firstBody = response.body | ||
| 21 | |||
| 22 | const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | ||
| 23 | let i = 0 | ||
| 24 | let nextLink = firstBody.first | ||
| 25 | while (nextLink && i < limit) { | ||
| 26 | let body: any | ||
| 27 | |||
| 28 | if (typeof nextLink === 'string') { | ||
| 29 | // Don't crawl ourselves | ||
| 30 | const remoteHost = new URL(nextLink).host | ||
| 31 | if (remoteHost === WEBSERVER.HOST) continue | ||
| 32 | |||
| 33 | url = nextLink | ||
| 34 | |||
| 35 | const res = await fetchAP<ActivityPubOrderedCollection<T>>(url) | ||
| 36 | body = res.body | ||
| 37 | } else { | ||
| 38 | // nextLink is already the object we want | ||
| 39 | body = nextLink | ||
| 40 | } | ||
| 41 | |||
| 42 | nextLink = body.next | ||
| 43 | i++ | ||
| 44 | |||
| 45 | if (Array.isArray(body.orderedItems)) { | ||
| 46 | const items = body.orderedItems | ||
| 47 | logger.info('Processing %i ActivityPub items for %s.', items.length, url) | ||
| 48 | |||
| 49 | await handler(items) | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | if (cleaner) await retryTransactionWrapper(cleaner, startDate) | ||
| 54 | } | ||
| 55 | |||
| 56 | export { | ||
| 57 | crawlCollectionPage | ||
| 58 | } | ||
diff --git a/server/lib/activitypub/follow.ts b/server/lib/activitypub/follow.ts deleted file mode 100644 index f6e2a48fd..000000000 --- a/server/lib/activitypub/follow.ts +++ /dev/null | |||
| @@ -1,51 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { getServerActor } from '@server/models/application/application' | ||
| 3 | import { logger } from '../../helpers/logger' | ||
| 4 | import { CONFIG } from '../../initializers/config' | ||
| 5 | import { SERVER_ACTOR_NAME } from '../../initializers/constants' | ||
| 6 | import { ServerModel } from '../../models/server/server' | ||
| 7 | import { MActorFollowActors } from '../../types/models' | ||
| 8 | import { JobQueue } from '../job-queue' | ||
| 9 | |||
| 10 | async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors, transaction?: Transaction) { | ||
| 11 | if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return | ||
| 12 | |||
| 13 | const follower = actorFollow.ActorFollower | ||
| 14 | |||
| 15 | if (follower.type === 'Application' && follower.preferredUsername === SERVER_ACTOR_NAME) { | ||
| 16 | logger.info('Auto follow back %s.', follower.url) | ||
| 17 | |||
| 18 | const me = await getServerActor() | ||
| 19 | |||
| 20 | const server = await ServerModel.load(follower.serverId, transaction) | ||
| 21 | const host = server.host | ||
| 22 | |||
| 23 | const payload = { | ||
| 24 | host, | ||
| 25 | name: SERVER_ACTOR_NAME, | ||
| 26 | followerActorId: me.id, | ||
| 27 | isAutoFollow: true | ||
| 28 | } | ||
| 29 | |||
| 30 | JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) | ||
| 31 | } | ||
| 32 | } | ||
| 33 | |||
| 34 | // If we only have an host, use a default account handle | ||
| 35 | function getRemoteNameAndHost (handleOrHost: string) { | ||
| 36 | let name = SERVER_ACTOR_NAME | ||
| 37 | let host = handleOrHost | ||
| 38 | |||
| 39 | const splitted = handleOrHost.split('@') | ||
| 40 | if (splitted.length === 2) { | ||
| 41 | name = splitted[0] | ||
| 42 | host = splitted[1] | ||
| 43 | } | ||
| 44 | |||
| 45 | return { name, host } | ||
| 46 | } | ||
| 47 | |||
| 48 | export { | ||
| 49 | autoFollowBackIfNeeded, | ||
| 50 | getRemoteNameAndHost | ||
| 51 | } | ||
diff --git a/server/lib/activitypub/inbox-manager.ts b/server/lib/activitypub/inbox-manager.ts deleted file mode 100644 index 27778cc9d..000000000 --- a/server/lib/activitypub/inbox-manager.ts +++ /dev/null | |||
| @@ -1,47 +0,0 @@ | |||
| 1 | import PQueue from 'p-queue' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' | ||
| 4 | import { MActorDefault, MActorSignature } from '@server/types/models' | ||
| 5 | import { Activity } from '@shared/models' | ||
| 6 | import { StatsManager } from '../stat-manager' | ||
| 7 | import { processActivities } from './process' | ||
| 8 | |||
| 9 | class InboxManager { | ||
| 10 | |||
| 11 | private static instance: InboxManager | ||
| 12 | private readonly inboxQueue: PQueue | ||
| 13 | |||
| 14 | private constructor () { | ||
| 15 | this.inboxQueue = new PQueue({ concurrency: 1 }) | ||
| 16 | |||
| 17 | setInterval(() => { | ||
| 18 | StatsManager.Instance.updateInboxWaiting(this.getActivityPubMessagesWaiting()) | ||
| 19 | }, SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS) | ||
| 20 | } | ||
| 21 | |||
| 22 | addInboxMessage (param: { | ||
| 23 | activities: Activity[] | ||
| 24 | signatureActor?: MActorSignature | ||
| 25 | inboxActor?: MActorDefault | ||
| 26 | }) { | ||
| 27 | this.inboxQueue.add(() => { | ||
| 28 | const options = { signatureActor: param.signatureActor, inboxActor: param.inboxActor } | ||
| 29 | |||
| 30 | return processActivities(param.activities, options) | ||
| 31 | }).catch(err => logger.error('Error with inbox queue.', { err })) | ||
| 32 | } | ||
| 33 | |||
| 34 | getActivityPubMessagesWaiting () { | ||
| 35 | return this.inboxQueue.size + this.inboxQueue.pending | ||
| 36 | } | ||
| 37 | |||
| 38 | static get Instance () { | ||
| 39 | return this.instance || (this.instance = new this()) | ||
| 40 | } | ||
| 41 | } | ||
| 42 | |||
| 43 | // --------------------------------------------------------------------------- | ||
| 44 | |||
| 45 | export { | ||
| 46 | InboxManager | ||
| 47 | } | ||
diff --git a/server/lib/activitypub/local-video-viewer.ts b/server/lib/activitypub/local-video-viewer.ts deleted file mode 100644 index bdd746791..000000000 --- a/server/lib/activitypub/local-video-viewer.ts +++ /dev/null | |||
| @@ -1,44 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
| 3 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' | ||
| 4 | import { MVideo } from '@server/types/models' | ||
| 5 | import { WatchActionObject } from '@shared/models' | ||
| 6 | import { getDurationFromActivityStream } from './activity' | ||
| 7 | |||
| 8 | async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) { | ||
| 9 | const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id) | ||
| 10 | if (stats) await stats.destroy({ transaction: t }) | ||
| 11 | |||
| 12 | const localVideoViewer = await LocalVideoViewerModel.create({ | ||
| 13 | url: watchAction.id, | ||
| 14 | uuid: watchAction.uuid, | ||
| 15 | |||
| 16 | watchTime: getDurationFromActivityStream(watchAction.duration), | ||
| 17 | |||
| 18 | startDate: new Date(watchAction.startTime), | ||
| 19 | endDate: new Date(watchAction.endTime), | ||
| 20 | |||
| 21 | country: watchAction.location | ||
| 22 | ? watchAction.location.addressCountry | ||
| 23 | : null, | ||
| 24 | |||
| 25 | videoId: video.id | ||
| 26 | }, { transaction: t }) | ||
| 27 | |||
| 28 | await LocalVideoViewerWatchSectionModel.bulkCreateSections({ | ||
| 29 | localVideoViewerId: localVideoViewer.id, | ||
| 30 | |||
| 31 | watchSections: watchAction.watchSections.map(s => ({ | ||
| 32 | start: s.startTimestamp, | ||
| 33 | end: s.endTimestamp | ||
| 34 | })), | ||
| 35 | |||
| 36 | transaction: t | ||
| 37 | }) | ||
| 38 | } | ||
| 39 | |||
| 40 | // --------------------------------------------------------------------------- | ||
| 41 | |||
| 42 | export { | ||
| 43 | createOrUpdateLocalVideoViewer | ||
| 44 | } | ||
diff --git a/server/lib/activitypub/outbox.ts b/server/lib/activitypub/outbox.ts deleted file mode 100644 index 5eef76871..000000000 --- a/server/lib/activitypub/outbox.ts +++ /dev/null | |||
| @@ -1,24 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { ActorModel } from '@server/models/actor/actor' | ||
| 3 | import { getServerActor } from '@server/models/application/application' | ||
| 4 | import { JobQueue } from '../job-queue' | ||
| 5 | |||
| 6 | async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) { | ||
| 7 | // Don't fetch ourselves | ||
| 8 | const serverActor = await getServerActor() | ||
| 9 | if (serverActor.id === actor.id) { | ||
| 10 | logger.error('Cannot fetch our own outbox!') | ||
| 11 | return undefined | ||
| 12 | } | ||
| 13 | |||
| 14 | const payload = { | ||
| 15 | uri: actor.outboxUrl, | ||
| 16 | type: 'activity' as 'activity' | ||
| 17 | } | ||
| 18 | |||
| 19 | return JobQueue.Instance.createJobAsync({ type: 'activitypub-http-fetcher', payload }) | ||
| 20 | } | ||
| 21 | |||
| 22 | export { | ||
| 23 | addFetchOutboxJob | ||
| 24 | } | ||
diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts deleted file mode 100644 index b24299f29..000000000 --- a/server/lib/activitypub/playlists/create-update.ts +++ /dev/null | |||
| @@ -1,157 +0,0 @@ | |||
| 1 | import { map } from 'bluebird' | ||
| 2 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
| 3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 5 | import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' | ||
| 6 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 7 | import { updateRemotePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' | ||
| 8 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | ||
| 9 | import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' | ||
| 10 | import { FilteredModelAttributes } from '@server/types' | ||
| 11 | import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models' | ||
| 12 | import { PlaylistObject } from '@shared/models' | ||
| 13 | import { AttributesOnly } from '@shared/typescript-utils' | ||
| 14 | import { getAPId } from '../activity' | ||
| 15 | import { getOrCreateAPActor } from '../actors' | ||
| 16 | import { crawlCollectionPage } from '../crawl' | ||
| 17 | import { getOrCreateAPVideo } from '../videos' | ||
| 18 | import { | ||
| 19 | fetchRemotePlaylistElement, | ||
| 20 | fetchRemoteVideoPlaylist, | ||
| 21 | playlistElementObjectToDBAttributes, | ||
| 22 | playlistObjectToDBAttributes | ||
| 23 | } from './shared' | ||
| 24 | |||
| 25 | const lTags = loggerTagsFactory('ap', 'video-playlist') | ||
| 26 | |||
| 27 | async function createAccountPlaylists (playlistUrls: string[]) { | ||
| 28 | await map(playlistUrls, async playlistUrl => { | ||
| 29 | try { | ||
| 30 | const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) | ||
| 31 | if (exists === true) return | ||
| 32 | |||
| 33 | const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) | ||
| 34 | |||
| 35 | if (playlistObject === undefined) { | ||
| 36 | throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) | ||
| 37 | } | ||
| 38 | |||
| 39 | return createOrUpdateVideoPlaylist(playlistObject) | ||
| 40 | } catch (err) { | ||
| 41 | logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) | ||
| 42 | } | ||
| 43 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
| 44 | } | ||
| 45 | |||
| 46 | async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) { | ||
| 47 | const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to) | ||
| 48 | |||
| 49 | await setVideoChannel(playlistObject, playlistAttributes) | ||
| 50 | |||
| 51 | const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true }) | ||
| 52 | |||
| 53 | const playlistElementUrls = await fetchElementUrls(playlistObject) | ||
| 54 | |||
| 55 | // Refetch playlist from DB since elements fetching could be long in time | ||
| 56 | const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null) | ||
| 57 | |||
| 58 | await updatePlaylistThumbnail(playlistObject, playlist) | ||
| 59 | |||
| 60 | const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist) | ||
| 61 | playlist.setVideosLength(elementsLength) | ||
| 62 | |||
| 63 | return playlist | ||
| 64 | } | ||
| 65 | |||
| 66 | // --------------------------------------------------------------------------- | ||
| 67 | |||
| 68 | export { | ||
| 69 | createAccountPlaylists, | ||
| 70 | createOrUpdateVideoPlaylist | ||
| 71 | } | ||
| 72 | |||
| 73 | // --------------------------------------------------------------------------- | ||
| 74 | |||
| 75 | async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) { | ||
| 76 | if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) { | ||
| 77 | throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) | ||
| 78 | } | ||
| 79 | |||
| 80 | const actor = await getOrCreateAPActor(getAPId(playlistObject.attributedTo[0]), 'all') | ||
| 81 | |||
| 82 | if (!actor.VideoChannel) { | ||
| 83 | logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) | ||
| 84 | return | ||
| 85 | } | ||
| 86 | |||
| 87 | playlistAttributes.videoChannelId = actor.VideoChannel.id | ||
| 88 | playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id | ||
| 89 | } | ||
| 90 | |||
| 91 | async function fetchElementUrls (playlistObject: PlaylistObject) { | ||
| 92 | let accItems: string[] = [] | ||
| 93 | await crawlCollectionPage<string>(playlistObject.id, items => { | ||
| 94 | accItems = accItems.concat(items) | ||
| 95 | |||
| 96 | return Promise.resolve() | ||
| 97 | }) | ||
| 98 | |||
| 99 | return accItems | ||
| 100 | } | ||
| 101 | |||
| 102 | async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) { | ||
| 103 | if (playlistObject.icon) { | ||
| 104 | let thumbnailModel: MThumbnail | ||
| 105 | |||
| 106 | try { | ||
| 107 | thumbnailModel = await updateRemotePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) | ||
| 108 | await playlist.setAndSaveThumbnail(thumbnailModel, undefined) | ||
| 109 | } catch (err) { | ||
| 110 | logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) | ||
| 111 | |||
| 112 | if (thumbnailModel) await thumbnailModel.removeThumbnail() | ||
| 113 | } | ||
| 114 | |||
| 115 | return | ||
| 116 | } | ||
| 117 | |||
| 118 | // Playlist does not have an icon, destroy existing one | ||
| 119 | if (playlist.hasThumbnail()) { | ||
| 120 | await playlist.Thumbnail.destroy() | ||
| 121 | playlist.Thumbnail = null | ||
| 122 | } | ||
| 123 | } | ||
| 124 | |||
| 125 | async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { | ||
| 126 | const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist) | ||
| 127 | |||
| 128 | await retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { | ||
| 129 | await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) | ||
| 130 | |||
| 131 | for (const element of elementsToCreate) { | ||
| 132 | await VideoPlaylistElementModel.create(element, { transaction: t }) | ||
| 133 | } | ||
| 134 | })) | ||
| 135 | |||
| 136 | logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) | ||
| 137 | |||
| 138 | return elementsToCreate.length | ||
| 139 | } | ||
| 140 | |||
| 141 | async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { | ||
| 142 | const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = [] | ||
| 143 | |||
| 144 | await map(elementUrls, async elementUrl => { | ||
| 145 | try { | ||
| 146 | const { elementObject } = await fetchRemotePlaylistElement(elementUrl) | ||
| 147 | |||
| 148 | const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' }) | ||
| 149 | |||
| 150 | elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video)) | ||
| 151 | } catch (err) { | ||
| 152 | logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) }) | ||
| 153 | } | ||
| 154 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
| 155 | |||
| 156 | return elementsToCreate | ||
| 157 | } | ||
diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts deleted file mode 100644 index c34554d69..000000000 --- a/server/lib/activitypub/playlists/get.ts +++ /dev/null | |||
| @@ -1,35 +0,0 @@ | |||
| 1 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | ||
| 2 | import { MVideoPlaylistFullSummary } from '@server/types/models' | ||
| 3 | import { APObjectId } from '@shared/models' | ||
| 4 | import { getAPId } from '../activity' | ||
| 5 | import { createOrUpdateVideoPlaylist } from './create-update' | ||
| 6 | import { scheduleRefreshIfNeeded } from './refresh' | ||
| 7 | import { fetchRemoteVideoPlaylist } from './shared' | ||
| 8 | |||
| 9 | async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promise<MVideoPlaylistFullSummary> { | ||
| 10 | const playlistUrl = getAPId(playlistObjectArg) | ||
| 11 | |||
| 12 | const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) | ||
| 13 | |||
| 14 | if (playlistFromDatabase) { | ||
| 15 | scheduleRefreshIfNeeded(playlistFromDatabase) | ||
| 16 | |||
| 17 | return playlistFromDatabase | ||
| 18 | } | ||
| 19 | |||
| 20 | const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) | ||
| 21 | if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl) | ||
| 22 | |||
| 23 | // playlistUrl is just an alias/redirection, so process object id instead | ||
| 24 | if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject) | ||
| 25 | |||
| 26 | const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject) | ||
| 27 | |||
| 28 | return playlistCreated | ||
| 29 | } | ||
| 30 | |||
| 31 | // --------------------------------------------------------------------------- | ||
| 32 | |||
| 33 | export { | ||
| 34 | getOrCreateAPVideoPlaylist | ||
| 35 | } | ||
diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts deleted file mode 100644 index e2470a674..000000000 --- a/server/lib/activitypub/playlists/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './get' | ||
| 2 | export * from './create-update' | ||
| 3 | export * from './refresh' | ||
diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts deleted file mode 100644 index 33260ea02..000000000 --- a/server/lib/activitypub/playlists/refresh.ts +++ /dev/null | |||
| @@ -1,53 +0,0 @@ | |||
| 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 2 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
| 3 | import { JobQueue } from '@server/lib/job-queue' | ||
| 4 | import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models' | ||
| 5 | import { HttpStatusCode } from '@shared/models' | ||
| 6 | import { createOrUpdateVideoPlaylist } from './create-update' | ||
| 7 | import { fetchRemoteVideoPlaylist } from './shared' | ||
| 8 | |||
| 9 | function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) { | ||
| 10 | if (!playlist.isOutdated()) return | ||
| 11 | |||
| 12 | JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } }) | ||
| 13 | } | ||
| 14 | |||
| 15 | async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> { | ||
| 16 | if (!videoPlaylist.isOutdated()) return videoPlaylist | ||
| 17 | |||
| 18 | const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url) | ||
| 19 | |||
| 20 | logger.info('Refreshing playlist %s.', videoPlaylist.url, lTags()) | ||
| 21 | |||
| 22 | try { | ||
| 23 | const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) | ||
| 24 | |||
| 25 | if (playlistObject === undefined) { | ||
| 26 | logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags()) | ||
| 27 | |||
| 28 | await videoPlaylist.setAsRefreshed() | ||
| 29 | return videoPlaylist | ||
| 30 | } | ||
| 31 | |||
| 32 | await createOrUpdateVideoPlaylist(playlistObject) | ||
| 33 | |||
| 34 | return videoPlaylist | ||
| 35 | } catch (err) { | ||
| 36 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
| 37 | logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags()) | ||
| 38 | |||
| 39 | await videoPlaylist.destroy() | ||
| 40 | return undefined | ||
| 41 | } | ||
| 42 | |||
| 43 | logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() }) | ||
| 44 | |||
| 45 | await videoPlaylist.setAsRefreshed() | ||
| 46 | return videoPlaylist | ||
| 47 | } | ||
| 48 | } | ||
| 49 | |||
| 50 | export { | ||
| 51 | scheduleRefreshIfNeeded, | ||
| 52 | refreshVideoPlaylistIfNeeded | ||
| 53 | } | ||
diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts deleted file mode 100644 index a217f2291..000000000 --- a/server/lib/activitypub/playlists/shared/index.ts +++ /dev/null | |||
| @@ -1,2 +0,0 @@ | |||
| 1 | export * from './object-to-model-attributes' | ||
| 2 | export * from './url-to-object' | ||
diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts deleted file mode 100644 index 753b5e660..000000000 --- a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts +++ /dev/null | |||
| @@ -1,40 +0,0 @@ | |||
| 1 | import { ACTIVITY_PUB } from '@server/initializers/constants' | ||
| 2 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | ||
| 3 | import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' | ||
| 4 | import { MVideoId, MVideoPlaylistId } from '@server/types/models' | ||
| 5 | import { AttributesOnly } from '@shared/typescript-utils' | ||
| 6 | import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models' | ||
| 7 | |||
| 8 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) { | ||
| 9 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | ||
| 10 | ? VideoPlaylistPrivacy.PUBLIC | ||
| 11 | : VideoPlaylistPrivacy.UNLISTED | ||
| 12 | |||
| 13 | return { | ||
| 14 | name: playlistObject.name, | ||
| 15 | description: playlistObject.content, | ||
| 16 | privacy, | ||
| 17 | url: playlistObject.id, | ||
| 18 | uuid: playlistObject.uuid, | ||
| 19 | ownerAccountId: null, | ||
| 20 | videoChannelId: null, | ||
| 21 | createdAt: new Date(playlistObject.published), | ||
| 22 | updatedAt: new Date(playlistObject.updated) | ||
| 23 | } as AttributesOnly<VideoPlaylistModel> | ||
| 24 | } | ||
| 25 | |||
| 26 | function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { | ||
| 27 | return { | ||
| 28 | position: elementObject.position, | ||
| 29 | url: elementObject.id, | ||
| 30 | startTimestamp: elementObject.startTimestamp || null, | ||
| 31 | stopTimestamp: elementObject.stopTimestamp || null, | ||
| 32 | videoPlaylistId: videoPlaylist.id, | ||
| 33 | videoId: video.id | ||
| 34 | } as AttributesOnly<VideoPlaylistElementModel> | ||
| 35 | } | ||
| 36 | |||
| 37 | export { | ||
| 38 | playlistObjectToDBAttributes, | ||
| 39 | playlistElementObjectToDBAttributes | ||
| 40 | } | ||
diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts deleted file mode 100644 index fd9fe5558..000000000 --- a/server/lib/activitypub/playlists/shared/url-to-object.ts +++ /dev/null | |||
| @@ -1,47 +0,0 @@ | |||
| 1 | import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist' | ||
| 2 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
| 3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 4 | import { PlaylistElementObject, PlaylistObject } from '@shared/models' | ||
| 5 | import { fetchAP } from '../../activity' | ||
| 6 | import { checkUrlsSameHost } from '../../url' | ||
| 7 | |||
| 8 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { | ||
| 9 | const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl) | ||
| 10 | |||
| 11 | logger.info('Fetching remote playlist %s.', playlistUrl, lTags()) | ||
| 12 | |||
| 13 | const { body, statusCode } = await fetchAP<any>(playlistUrl) | ||
| 14 | |||
| 15 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { | ||
| 16 | logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() }) | ||
| 17 | return { statusCode, playlistObject: undefined } | ||
| 18 | } | ||
| 19 | |||
| 20 | if (!isArray(body.to)) { | ||
| 21 | logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() }) | ||
| 22 | return { statusCode, playlistObject: undefined } | ||
| 23 | } | ||
| 24 | |||
| 25 | return { statusCode, playlistObject: body } | ||
| 26 | } | ||
| 27 | |||
| 28 | async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> { | ||
| 29 | const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl) | ||
| 30 | |||
| 31 | logger.debug('Fetching remote playlist element %s.', elementUrl, lTags()) | ||
| 32 | |||
| 33 | const { body, statusCode } = await fetchAP<PlaylistElementObject>(elementUrl) | ||
| 34 | |||
| 35 | if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`) | ||
| 36 | |||
| 37 | if (checkUrlsSameHost(body.id, elementUrl) !== true) { | ||
| 38 | throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) | ||
| 39 | } | ||
| 40 | |||
| 41 | return { statusCode, elementObject: body } | ||
| 42 | } | ||
| 43 | |||
| 44 | export { | ||
| 45 | fetchRemoteVideoPlaylist, | ||
| 46 | fetchRemotePlaylistElement | ||
| 47 | } | ||
diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts deleted file mode 100644 index 5466739c1..000000000 --- a/server/lib/activitypub/process/index.ts +++ /dev/null | |||
| @@ -1 +0,0 @@ | |||
| 1 | export * from './process' | ||
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts deleted file mode 100644 index 077b01eda..000000000 --- a/server/lib/activitypub/process/process-accept.ts +++ /dev/null | |||
| @@ -1,32 +0,0 @@ | |||
| 1 | import { ActivityAccept } from '../../../../shared/models/activitypub' | ||
| 2 | import { ActorFollowModel } from '../../../models/actor/actor-follow' | ||
| 3 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 4 | import { MActorDefault, MActorSignature } from '../../../types/models' | ||
| 5 | import { addFetchOutboxJob } from '../outbox' | ||
| 6 | |||
| 7 | async function processAcceptActivity (options: APProcessorOptions<ActivityAccept>) { | ||
| 8 | const { byActor: targetActor, inboxActor } = options | ||
| 9 | if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') | ||
| 10 | |||
| 11 | return processAccept(inboxActor, targetActor) | ||
| 12 | } | ||
| 13 | |||
| 14 | // --------------------------------------------------------------------------- | ||
| 15 | |||
| 16 | export { | ||
| 17 | processAcceptActivity | ||
| 18 | } | ||
| 19 | |||
| 20 | // --------------------------------------------------------------------------- | ||
| 21 | |||
| 22 | async function processAccept (actor: MActorDefault, targetActor: MActorSignature) { | ||
| 23 | const follow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id) | ||
| 24 | if (!follow) throw new Error('Cannot find associated follow.') | ||
| 25 | |||
| 26 | if (follow.state !== 'accepted') { | ||
| 27 | follow.state = 'accepted' | ||
| 28 | await follow.save() | ||
| 29 | |||
| 30 | await addFetchOutboxJob(targetActor) | ||
| 31 | } | ||
| 32 | } | ||
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts deleted file mode 100644 index 9cc87ee27..000000000 --- a/server/lib/activitypub/process/process-announce.ts +++ /dev/null | |||
| @@ -1,75 +0,0 @@ | |||
| 1 | import { getAPId } from '@server/lib/activitypub/activity' | ||
| 2 | import { ActivityAnnounce } from '../../../../shared/models/activitypub' | ||
| 3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 4 | import { logger } from '../../../helpers/logger' | ||
| 5 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 6 | import { VideoShareModel } from '../../../models/video/video-share' | ||
| 7 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 8 | import { MActorSignature, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' | ||
| 9 | import { Notifier } from '../../notifier' | ||
| 10 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | ||
| 11 | import { getOrCreateAPVideo } from '../videos' | ||
| 12 | |||
| 13 | async function processAnnounceActivity (options: APProcessorOptions<ActivityAnnounce>) { | ||
| 14 | const { activity, byActor: actorAnnouncer } = options | ||
| 15 | // Only notify if it is not from a fetcher job | ||
| 16 | const notify = options.fromFetch !== true | ||
| 17 | |||
| 18 | // Announces on accounts are not supported | ||
| 19 | if (actorAnnouncer.type !== 'Application' && actorAnnouncer.type !== 'Group') return | ||
| 20 | |||
| 21 | return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity, notify) | ||
| 22 | } | ||
| 23 | |||
| 24 | // --------------------------------------------------------------------------- | ||
| 25 | |||
| 26 | export { | ||
| 27 | processAnnounceActivity | ||
| 28 | } | ||
| 29 | |||
| 30 | // --------------------------------------------------------------------------- | ||
| 31 | |||
| 32 | async function processVideoShare (actorAnnouncer: MActorSignature, activity: ActivityAnnounce, notify: boolean) { | ||
| 33 | const objectUri = getAPId(activity.object) | ||
| 34 | |||
| 35 | let video: MVideoAccountLightBlacklistAllFiles | ||
| 36 | let videoCreated: boolean | ||
| 37 | |||
| 38 | try { | ||
| 39 | const result = await getOrCreateAPVideo({ videoObject: objectUri }) | ||
| 40 | video = result.video | ||
| 41 | videoCreated = result.created | ||
| 42 | } catch (err) { | ||
| 43 | logger.debug('Cannot process share of %s. Maybe this is not a video object, so just skipping.', objectUri, { err }) | ||
| 44 | return | ||
| 45 | } | ||
| 46 | |||
| 47 | await sequelizeTypescript.transaction(async t => { | ||
| 48 | // Add share entry | ||
| 49 | |||
| 50 | const share = { | ||
| 51 | actorId: actorAnnouncer.id, | ||
| 52 | videoId: video.id, | ||
| 53 | url: activity.id | ||
| 54 | } | ||
| 55 | |||
| 56 | const [ , created ] = await VideoShareModel.findOrCreate({ | ||
| 57 | where: { | ||
| 58 | url: activity.id | ||
| 59 | }, | ||
| 60 | defaults: share, | ||
| 61 | transaction: t | ||
| 62 | }) | ||
| 63 | |||
| 64 | if (video.isOwned() && created === true) { | ||
| 65 | // Don't resend the activity to the sender | ||
| 66 | const exceptions = [ actorAnnouncer ] | ||
| 67 | |||
| 68 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | ||
| 69 | } | ||
| 70 | |||
| 71 | return undefined | ||
| 72 | }) | ||
| 73 | |||
| 74 | if (videoCreated && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
| 75 | } | ||
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts deleted file mode 100644 index 5f980de65..000000000 --- a/server/lib/activitypub/process/process-create.ts +++ /dev/null | |||
| @@ -1,170 +0,0 @@ | |||
| 1 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | ||
| 2 | import { isRedundancyAccepted } from '@server/lib/redundancy' | ||
| 3 | import { VideoModel } from '@server/models/video/video' | ||
| 4 | import { | ||
| 5 | AbuseObject, | ||
| 6 | ActivityCreate, | ||
| 7 | ActivityCreateObject, | ||
| 8 | ActivityObject, | ||
| 9 | CacheFileObject, | ||
| 10 | PlaylistObject, | ||
| 11 | VideoCommentObject, | ||
| 12 | VideoObject, | ||
| 13 | WatchActionObject | ||
| 14 | } from '@shared/models' | ||
| 15 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 16 | import { logger } from '../../../helpers/logger' | ||
| 17 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 18 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 19 | import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' | ||
| 20 | import { Notifier } from '../../notifier' | ||
| 21 | import { fetchAPObjectIfNeeded } from '../activity' | ||
| 22 | import { createOrUpdateCacheFile } from '../cache-file' | ||
| 23 | import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' | ||
| 24 | import { createOrUpdateVideoPlaylist } from '../playlists' | ||
| 25 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | ||
| 26 | import { resolveThread } from '../video-comments' | ||
| 27 | import { getOrCreateAPVideo } from '../videos' | ||
| 28 | |||
| 29 | async function processCreateActivity (options: APProcessorOptions<ActivityCreate<ActivityCreateObject>>) { | ||
| 30 | const { activity, byActor } = options | ||
| 31 | |||
| 32 | // Only notify if it is not from a fetcher job | ||
| 33 | const notify = options.fromFetch !== true | ||
| 34 | const activityObject = await fetchAPObjectIfNeeded<Exclude<ActivityObject, AbuseObject>>(activity.object) | ||
| 35 | const activityType = activityObject.type | ||
| 36 | |||
| 37 | if (activityType === 'Video') { | ||
| 38 | return processCreateVideo(activityObject, notify) | ||
| 39 | } | ||
| 40 | |||
| 41 | if (activityType === 'Note') { | ||
| 42 | // Comments will be fetched from videos | ||
| 43 | if (options.fromFetch) return | ||
| 44 | |||
| 45 | return retryTransactionWrapper(processCreateVideoComment, activity, activityObject, byActor, notify) | ||
| 46 | } | ||
| 47 | |||
| 48 | if (activityType === 'WatchAction') { | ||
| 49 | return retryTransactionWrapper(processCreateWatchAction, activityObject) | ||
| 50 | } | ||
| 51 | |||
| 52 | if (activityType === 'CacheFile') { | ||
| 53 | return retryTransactionWrapper(processCreateCacheFile, activity, activityObject, byActor) | ||
| 54 | } | ||
| 55 | |||
| 56 | if (activityType === 'Playlist') { | ||
| 57 | return retryTransactionWrapper(processCreatePlaylist, activity, activityObject, byActor) | ||
| 58 | } | ||
| 59 | |||
| 60 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) | ||
| 61 | return Promise.resolve(undefined) | ||
| 62 | } | ||
| 63 | |||
| 64 | // --------------------------------------------------------------------------- | ||
| 65 | |||
| 66 | export { | ||
| 67 | processCreateActivity | ||
| 68 | } | ||
| 69 | |||
| 70 | // --------------------------------------------------------------------------- | ||
| 71 | |||
| 72 | async function processCreateVideo (videoToCreateData: VideoObject, notify: boolean) { | ||
| 73 | const syncParam = { rates: false, shares: false, comments: false, refreshVideo: false } | ||
| 74 | const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) | ||
| 75 | |||
| 76 | if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
| 77 | |||
| 78 | return video | ||
| 79 | } | ||
| 80 | |||
| 81 | async function processCreateCacheFile ( | ||
| 82 | activity: ActivityCreate<CacheFileObject | string>, | ||
| 83 | cacheFile: CacheFileObject, | ||
| 84 | byActor: MActorSignature | ||
| 85 | ) { | ||
| 86 | if (await isRedundancyAccepted(activity, byActor) !== true) return | ||
| 87 | |||
| 88 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) | ||
| 89 | |||
| 90 | await sequelizeTypescript.transaction(async t => { | ||
| 91 | return createOrUpdateCacheFile(cacheFile, video, byActor, t) | ||
| 92 | }) | ||
| 93 | |||
| 94 | if (video.isOwned()) { | ||
| 95 | // Don't resend the activity to the sender | ||
| 96 | const exceptions = [ byActor ] | ||
| 97 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
| 98 | } | ||
| 99 | } | ||
| 100 | |||
| 101 | async function processCreateWatchAction (watchAction: WatchActionObject) { | ||
| 102 | if (watchAction.actionStatus !== 'CompletedActionStatus') return | ||
| 103 | |||
| 104 | const video = await VideoModel.loadByUrl(watchAction.object) | ||
| 105 | if (video.remote) return | ||
| 106 | |||
| 107 | await sequelizeTypescript.transaction(async t => { | ||
| 108 | return createOrUpdateLocalVideoViewer(watchAction, video, t) | ||
| 109 | }) | ||
| 110 | } | ||
| 111 | |||
| 112 | async function processCreateVideoComment ( | ||
| 113 | activity: ActivityCreate<VideoCommentObject | string>, | ||
| 114 | commentObject: VideoCommentObject, | ||
| 115 | byActor: MActorSignature, | ||
| 116 | notify: boolean | ||
| 117 | ) { | ||
| 118 | const byAccount = byActor.Account | ||
| 119 | |||
| 120 | if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) | ||
| 121 | |||
| 122 | let video: MVideoAccountLightBlacklistAllFiles | ||
| 123 | let created: boolean | ||
| 124 | let comment: MCommentOwnerVideo | ||
| 125 | |||
| 126 | try { | ||
| 127 | const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) | ||
| 128 | if (!resolveThreadResult) return // Comment not accepted | ||
| 129 | |||
| 130 | video = resolveThreadResult.video | ||
| 131 | created = resolveThreadResult.commentCreated | ||
| 132 | comment = resolveThreadResult.comment | ||
| 133 | } catch (err) { | ||
| 134 | logger.debug( | ||
| 135 | 'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.', | ||
| 136 | commentObject.inReplyTo, | ||
| 137 | { err } | ||
| 138 | ) | ||
| 139 | return | ||
| 140 | } | ||
| 141 | |||
| 142 | // Try to not forward unwanted comments on our videos | ||
| 143 | if (video.isOwned()) { | ||
| 144 | if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) { | ||
| 145 | logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url) | ||
| 146 | return | ||
| 147 | } | ||
| 148 | |||
| 149 | if (created === true) { | ||
| 150 | // Don't resend the activity to the sender | ||
| 151 | const exceptions = [ byActor ] | ||
| 152 | |||
| 153 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
| 154 | } | ||
| 155 | } | ||
| 156 | |||
| 157 | if (created && notify) Notifier.Instance.notifyOnNewComment(comment) | ||
| 158 | } | ||
| 159 | |||
| 160 | async function processCreatePlaylist ( | ||
| 161 | activity: ActivityCreate<PlaylistObject | string>, | ||
| 162 | playlistObject: PlaylistObject, | ||
| 163 | byActor: MActorSignature | ||
| 164 | ) { | ||
| 165 | const byAccount = byActor.Account | ||
| 166 | |||
| 167 | if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) | ||
| 168 | |||
| 169 | await createOrUpdateVideoPlaylist(playlistObject, activity.to) | ||
| 170 | } | ||
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts deleted file mode 100644 index ac0e7e235..000000000 --- a/server/lib/activitypub/process/process-delete.ts +++ /dev/null | |||
| @@ -1,153 +0,0 @@ | |||
| 1 | import { ActivityDelete } from '../../../../shared/models/activitypub' | ||
| 2 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 5 | import { ActorModel } from '../../../models/actor/actor' | ||
| 6 | import { VideoModel } from '../../../models/video/video' | ||
| 7 | import { VideoCommentModel } from '../../../models/video/video-comment' | ||
| 8 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
| 9 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 10 | import { | ||
| 11 | MAccountActor, | ||
| 12 | MActor, | ||
| 13 | MActorFull, | ||
| 14 | MActorSignature, | ||
| 15 | MChannelAccountActor, | ||
| 16 | MChannelActor, | ||
| 17 | MCommentOwnerVideo | ||
| 18 | } from '../../../types/models' | ||
| 19 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | ||
| 20 | |||
| 21 | async function processDeleteActivity (options: APProcessorOptions<ActivityDelete>) { | ||
| 22 | const { activity, byActor } = options | ||
| 23 | |||
| 24 | const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id | ||
| 25 | |||
| 26 | if (activity.actor === objectUrl) { | ||
| 27 | // We need more attributes (all the account and channel) | ||
| 28 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) | ||
| 29 | |||
| 30 | if (byActorFull.type === 'Person') { | ||
| 31 | if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.') | ||
| 32 | |||
| 33 | const accountToDelete = byActorFull.Account as MAccountActor | ||
| 34 | accountToDelete.Actor = byActorFull | ||
| 35 | |||
| 36 | return retryTransactionWrapper(processDeleteAccount, accountToDelete) | ||
| 37 | } else if (byActorFull.type === 'Group') { | ||
| 38 | if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.') | ||
| 39 | |||
| 40 | const channelToDelete = byActorFull.VideoChannel as MChannelAccountActor & { Actor: MActorFull } | ||
| 41 | channelToDelete.Actor = byActorFull | ||
| 42 | return retryTransactionWrapper(processDeleteVideoChannel, channelToDelete) | ||
| 43 | } | ||
| 44 | } | ||
| 45 | |||
| 46 | { | ||
| 47 | const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(objectUrl) | ||
| 48 | if (videoCommentInstance) { | ||
| 49 | return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity) | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | { | ||
| 54 | const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) | ||
| 55 | if (videoInstance) { | ||
| 56 | if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`) | ||
| 57 | |||
| 58 | return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance) | ||
| 59 | } | ||
| 60 | } | ||
| 61 | |||
| 62 | { | ||
| 63 | const videoPlaylist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(objectUrl) | ||
| 64 | if (videoPlaylist) { | ||
| 65 | if (videoPlaylist.isOwned()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`) | ||
| 66 | |||
| 67 | return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist) | ||
| 68 | } | ||
| 69 | } | ||
| 70 | |||
| 71 | return undefined | ||
| 72 | } | ||
| 73 | |||
| 74 | // --------------------------------------------------------------------------- | ||
| 75 | |||
| 76 | export { | ||
| 77 | processDeleteActivity | ||
| 78 | } | ||
| 79 | |||
| 80 | // --------------------------------------------------------------------------- | ||
| 81 | |||
| 82 | async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) { | ||
| 83 | logger.debug('Removing remote video "%s".', videoToDelete.uuid) | ||
| 84 | |||
| 85 | await sequelizeTypescript.transaction(async t => { | ||
| 86 | if (videoToDelete.VideoChannel.Account.Actor.id !== actor.id) { | ||
| 87 | throw new Error('Account ' + actor.url + ' does not own video channel ' + videoToDelete.VideoChannel.Actor.url) | ||
| 88 | } | ||
| 89 | |||
| 90 | await videoToDelete.destroy({ transaction: t }) | ||
| 91 | }) | ||
| 92 | |||
| 93 | logger.info('Remote video with uuid %s removed.', videoToDelete.uuid) | ||
| 94 | } | ||
| 95 | |||
| 96 | async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: VideoPlaylistModel) { | ||
| 97 | logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid) | ||
| 98 | |||
| 99 | await sequelizeTypescript.transaction(async t => { | ||
| 100 | if (playlistToDelete.OwnerAccount.Actor.id !== actor.id) { | ||
| 101 | throw new Error('Account ' + actor.url + ' does not own video playlist ' + playlistToDelete.url) | ||
| 102 | } | ||
| 103 | |||
| 104 | await playlistToDelete.destroy({ transaction: t }) | ||
| 105 | }) | ||
| 106 | |||
| 107 | logger.info('Remote video playlist with uuid %s removed.', playlistToDelete.uuid) | ||
| 108 | } | ||
| 109 | |||
| 110 | async function processDeleteAccount (accountToRemove: MAccountActor) { | ||
| 111 | logger.debug('Removing remote account "%s".', accountToRemove.Actor.url) | ||
| 112 | |||
| 113 | await sequelizeTypescript.transaction(async t => { | ||
| 114 | await accountToRemove.destroy({ transaction: t }) | ||
| 115 | }) | ||
| 116 | |||
| 117 | logger.info('Remote account %s removed.', accountToRemove.Actor.url) | ||
| 118 | } | ||
| 119 | |||
| 120 | async function processDeleteVideoChannel (videoChannelToRemove: MChannelActor) { | ||
| 121 | logger.debug('Removing remote video channel "%s".', videoChannelToRemove.Actor.url) | ||
| 122 | |||
| 123 | await sequelizeTypescript.transaction(async t => { | ||
| 124 | await videoChannelToRemove.destroy({ transaction: t }) | ||
| 125 | }) | ||
| 126 | |||
| 127 | logger.info('Remote video channel %s removed.', videoChannelToRemove.Actor.url) | ||
| 128 | } | ||
| 129 | |||
| 130 | function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { | ||
| 131 | // Already deleted | ||
| 132 | if (videoComment.isDeleted()) return Promise.resolve() | ||
| 133 | |||
| 134 | logger.debug('Removing remote video comment "%s".', videoComment.url) | ||
| 135 | |||
| 136 | return sequelizeTypescript.transaction(async t => { | ||
| 137 | if (byActor.Account.id !== videoComment.Account.id && byActor.Account.id !== videoComment.Video.VideoChannel.accountId) { | ||
| 138 | throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) | ||
| 139 | } | ||
| 140 | |||
| 141 | videoComment.markAsDeleted() | ||
| 142 | |||
| 143 | await videoComment.save({ transaction: t }) | ||
| 144 | |||
| 145 | if (videoComment.Video.isOwned()) { | ||
| 146 | // Don't resend the activity to the sender | ||
| 147 | const exceptions = [ byActor ] | ||
| 148 | await forwardVideoRelatedActivity(activity, t, exceptions, videoComment.Video) | ||
| 149 | } | ||
| 150 | |||
| 151 | logger.info('Remote video comment %s removed.', videoComment.url) | ||
| 152 | }) | ||
| 153 | } | ||
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts deleted file mode 100644 index 4e270f917..000000000 --- a/server/lib/activitypub/process/process-dislike.ts +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | import { VideoModel } from '@server/models/video/video' | ||
| 2 | import { ActivityDislike } from '@shared/models' | ||
| 3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 4 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 5 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
| 6 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 7 | import { MActorSignature } from '../../../types/models' | ||
| 8 | import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' | ||
| 9 | |||
| 10 | async function processDislikeActivity (options: APProcessorOptions<ActivityDislike>) { | ||
| 11 | const { activity, byActor } = options | ||
| 12 | return retryTransactionWrapper(processDislike, activity, byActor) | ||
| 13 | } | ||
| 14 | |||
| 15 | // --------------------------------------------------------------------------- | ||
| 16 | |||
| 17 | export { | ||
| 18 | processDislikeActivity | ||
| 19 | } | ||
| 20 | |||
| 21 | // --------------------------------------------------------------------------- | ||
| 22 | |||
| 23 | async function processDislike (activity: ActivityDislike, byActor: MActorSignature) { | ||
| 24 | const dislikeObject = activity.object | ||
| 25 | const byAccount = byActor.Account | ||
| 26 | |||
| 27 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
| 28 | |||
| 29 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeObject, fetchType: 'only-video' }) | ||
| 30 | |||
| 31 | // We don't care about dislikes of remote videos | ||
| 32 | if (!onlyVideo.isOwned()) return | ||
| 33 | |||
| 34 | return sequelizeTypescript.transaction(async t => { | ||
| 35 | const video = await VideoModel.loadFull(onlyVideo.id, t) | ||
| 36 | |||
| 37 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) | ||
| 38 | if (existingRate && existingRate.type === 'dislike') return | ||
| 39 | |||
| 40 | await video.increment('dislikes', { transaction: t }) | ||
| 41 | video.dislikes++ | ||
| 42 | |||
| 43 | if (existingRate && existingRate.type === 'like') { | ||
| 44 | await video.decrement('likes', { transaction: t }) | ||
| 45 | video.likes-- | ||
| 46 | } | ||
| 47 | |||
| 48 | const rate = existingRate || new AccountVideoRateModel() | ||
| 49 | rate.type = 'dislike' | ||
| 50 | rate.videoId = video.id | ||
| 51 | rate.accountId = byAccount.id | ||
| 52 | rate.url = activity.id | ||
| 53 | |||
| 54 | await rate.save({ transaction: t }) | ||
| 55 | |||
| 56 | await federateVideoIfNeeded(video, false, t) | ||
| 57 | }) | ||
| 58 | } | ||
diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts deleted file mode 100644 index bea285670..000000000 --- a/server/lib/activitypub/process/process-flag.ts +++ /dev/null | |||
| @@ -1,103 +0,0 @@ | |||
| 1 | import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation' | ||
| 2 | import { AccountModel } from '@server/models/account/account' | ||
| 3 | import { VideoModel } from '@server/models/video/video' | ||
| 4 | import { VideoCommentModel } from '@server/models/video/video-comment' | ||
| 5 | import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' | ||
| 6 | import { AbuseState, ActivityFlag } from '@shared/models' | ||
| 7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 8 | import { logger } from '../../../helpers/logger' | ||
| 9 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 10 | import { getAPId } from '../../../lib/activitypub/activity' | ||
| 11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 12 | import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../types/models' | ||
| 13 | |||
| 14 | async function processFlagActivity (options: APProcessorOptions<ActivityFlag>) { | ||
| 15 | const { activity, byActor } = options | ||
| 16 | |||
| 17 | return retryTransactionWrapper(processCreateAbuse, activity, byActor) | ||
| 18 | } | ||
| 19 | |||
| 20 | // --------------------------------------------------------------------------- | ||
| 21 | |||
| 22 | export { | ||
| 23 | processFlagActivity | ||
| 24 | } | ||
| 25 | |||
| 26 | // --------------------------------------------------------------------------- | ||
| 27 | |||
| 28 | async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature) { | ||
| 29 | const account = byActor.Account | ||
| 30 | if (!account) throw new Error('Cannot create abuse with the non account actor ' + byActor.url) | ||
| 31 | |||
| 32 | const reporterAccount = await AccountModel.load(account.id) | ||
| 33 | |||
| 34 | const objects = Array.isArray(flag.object) ? flag.object : [ flag.object ] | ||
| 35 | |||
| 36 | const tags = Array.isArray(flag.tag) ? flag.tag : [] | ||
| 37 | const predefinedReasons = tags.map(tag => abusePredefinedReasonsMap[tag.name]) | ||
| 38 | .filter(v => !isNaN(v)) | ||
| 39 | |||
| 40 | const startAt = flag.startAt | ||
| 41 | const endAt = flag.endAt | ||
| 42 | |||
| 43 | for (const object of objects) { | ||
| 44 | try { | ||
| 45 | const uri = getAPId(object) | ||
| 46 | |||
| 47 | logger.debug('Reporting remote abuse for object %s.', uri) | ||
| 48 | |||
| 49 | await sequelizeTypescript.transaction(async t => { | ||
| 50 | const video = await VideoModel.loadByUrlAndPopulateAccount(uri, t) | ||
| 51 | let videoComment: MCommentOwnerVideo | ||
| 52 | let flaggedAccount: MAccountDefault | ||
| 53 | |||
| 54 | if (!video) videoComment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(uri, t) | ||
| 55 | if (!videoComment) flaggedAccount = await AccountModel.loadByUrl(uri, t) | ||
| 56 | |||
| 57 | if (!video && !videoComment && !flaggedAccount) { | ||
| 58 | logger.warn('Cannot flag unknown entity %s.', object) | ||
| 59 | return | ||
| 60 | } | ||
| 61 | |||
| 62 | const baseAbuse = { | ||
| 63 | reporterAccountId: reporterAccount.id, | ||
| 64 | reason: flag.content, | ||
| 65 | state: AbuseState.PENDING, | ||
| 66 | predefinedReasons | ||
| 67 | } | ||
| 68 | |||
| 69 | if (video) { | ||
| 70 | return createVideoAbuse({ | ||
| 71 | baseAbuse, | ||
| 72 | startAt, | ||
| 73 | endAt, | ||
| 74 | reporterAccount, | ||
| 75 | transaction: t, | ||
| 76 | videoInstance: video, | ||
| 77 | skipNotification: false | ||
| 78 | }) | ||
| 79 | } | ||
| 80 | |||
| 81 | if (videoComment) { | ||
| 82 | return createVideoCommentAbuse({ | ||
| 83 | baseAbuse, | ||
| 84 | reporterAccount, | ||
| 85 | transaction: t, | ||
| 86 | commentInstance: videoComment, | ||
| 87 | skipNotification: false | ||
| 88 | }) | ||
| 89 | } | ||
| 90 | |||
| 91 | return await createAccountAbuse({ | ||
| 92 | baseAbuse, | ||
| 93 | reporterAccount, | ||
| 94 | transaction: t, | ||
| 95 | accountInstance: flaggedAccount, | ||
| 96 | skipNotification: false | ||
| 97 | }) | ||
| 98 | }) | ||
| 99 | } catch (err) { | ||
| 100 | logger.debug('Cannot process report of %s', getAPId(object), { err }) | ||
| 101 | } | ||
| 102 | } | ||
| 103 | } | ||
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts deleted file mode 100644 index 7def753d5..000000000 --- a/server/lib/activitypub/process/process-follow.ts +++ /dev/null | |||
| @@ -1,156 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize/types' | ||
| 2 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | ||
| 3 | import { AccountModel } from '@server/models/account/account' | ||
| 4 | import { getServerActor } from '@server/models/application/application' | ||
| 5 | import { ActivityFollow } from '../../../../shared/models/activitypub' | ||
| 6 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 7 | import { logger } from '../../../helpers/logger' | ||
| 8 | import { CONFIG } from '../../../initializers/config' | ||
| 9 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 10 | import { getAPId } from '../../../lib/activitypub/activity' | ||
| 11 | import { ActorModel } from '../../../models/actor/actor' | ||
| 12 | import { ActorFollowModel } from '../../../models/actor/actor-follow' | ||
| 13 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 14 | import { MActorFollow, MActorFull, MActorId, MActorSignature } from '../../../types/models' | ||
| 15 | import { Notifier } from '../../notifier' | ||
| 16 | import { autoFollowBackIfNeeded } from '../follow' | ||
| 17 | import { sendAccept, sendReject } from '../send' | ||
| 18 | |||
| 19 | async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) { | ||
| 20 | const { activity, byActor } = options | ||
| 21 | |||
| 22 | const activityId = activity.id | ||
| 23 | const objectId = getAPId(activity.object) | ||
| 24 | |||
| 25 | return retryTransactionWrapper(processFollow, byActor, activityId, objectId) | ||
| 26 | } | ||
| 27 | |||
| 28 | // --------------------------------------------------------------------------- | ||
| 29 | |||
| 30 | export { | ||
| 31 | processFollowActivity | ||
| 32 | } | ||
| 33 | |||
| 34 | // --------------------------------------------------------------------------- | ||
| 35 | |||
| 36 | async function processFollow (byActor: MActorSignature, activityId: string, targetActorURL: string) { | ||
| 37 | const { actorFollow, created, targetActor } = await sequelizeTypescript.transaction(async t => { | ||
| 38 | const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) | ||
| 39 | |||
| 40 | if (!targetActor) throw new Error('Unknown actor') | ||
| 41 | if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') | ||
| 42 | |||
| 43 | if (await rejectIfInstanceFollowDisabled(byActor, activityId, targetActor)) return { actorFollow: undefined } | ||
| 44 | if (await rejectIfMuted(byActor, activityId, targetActor)) return { actorFollow: undefined } | ||
| 45 | |||
| 46 | const [ actorFollow, created ] = await ActorFollowModel.findOrCreateCustom({ | ||
| 47 | byActor, | ||
| 48 | targetActor, | ||
| 49 | activityId, | ||
| 50 | state: await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL | ||
| 51 | ? 'pending' | ||
| 52 | : 'accepted', | ||
| 53 | transaction: t | ||
| 54 | }) | ||
| 55 | |||
| 56 | if (rejectIfAlreadyRejected(actorFollow, byActor, activityId, targetActor)) return { actorFollow: undefined } | ||
| 57 | |||
| 58 | await acceptIfNeeded(actorFollow, targetActor, t) | ||
| 59 | |||
| 60 | await fixFollowURLIfNeeded(actorFollow, activityId, t) | ||
| 61 | |||
| 62 | actorFollow.ActorFollower = byActor | ||
| 63 | actorFollow.ActorFollowing = targetActor | ||
| 64 | |||
| 65 | // Target sends to actor he accepted the follow request | ||
| 66 | if (actorFollow.state === 'accepted') { | ||
| 67 | sendAccept(actorFollow) | ||
| 68 | |||
| 69 | await autoFollowBackIfNeeded(actorFollow, t) | ||
| 70 | } | ||
| 71 | |||
| 72 | return { actorFollow, created, targetActor } | ||
| 73 | }) | ||
| 74 | |||
| 75 | // Rejected | ||
| 76 | if (!actorFollow) return | ||
| 77 | |||
| 78 | if (created) { | ||
| 79 | const follower = await ActorModel.loadFull(byActor.id) | ||
| 80 | const actorFollowFull = Object.assign(actorFollow, { ActorFollowing: targetActor, ActorFollower: follower }) | ||
| 81 | |||
| 82 | if (await isFollowingInstance(targetActor)) { | ||
| 83 | Notifier.Instance.notifyOfNewInstanceFollow(actorFollowFull) | ||
| 84 | } else { | ||
| 85 | Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) | ||
| 86 | } | ||
| 87 | } | ||
| 88 | |||
| 89 | logger.info('Actor %s is followed by actor %s.', targetActorURL, byActor.url) | ||
| 90 | } | ||
| 91 | |||
| 92 | async function rejectIfInstanceFollowDisabled (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { | ||
| 93 | if (await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { | ||
| 94 | logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) | ||
| 95 | |||
| 96 | sendReject(activityId, byActor, targetActor) | ||
| 97 | |||
| 98 | return true | ||
| 99 | } | ||
| 100 | |||
| 101 | return false | ||
| 102 | } | ||
| 103 | |||
| 104 | async function rejectIfMuted (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { | ||
| 105 | const followerAccount = await AccountModel.load(byActor.Account.id) | ||
| 106 | const followingAccountId = targetActor.Account | ||
| 107 | |||
| 108 | if (followerAccount && await isBlockedByServerOrAccount(followerAccount, followingAccountId)) { | ||
| 109 | logger.info('Rejecting %s because follower is muted.', byActor.url) | ||
| 110 | |||
| 111 | sendReject(activityId, byActor, targetActor) | ||
| 112 | |||
| 113 | return true | ||
| 114 | } | ||
| 115 | |||
| 116 | return false | ||
| 117 | } | ||
| 118 | |||
| 119 | function rejectIfAlreadyRejected (actorFollow: MActorFollow, byActor: MActorSignature, activityId: string, targetActor: MActorFull) { | ||
| 120 | // Already rejected | ||
| 121 | if (actorFollow.state === 'rejected') { | ||
| 122 | logger.info('Rejecting %s because follow is already rejected.', byActor.url) | ||
| 123 | |||
| 124 | sendReject(activityId, byActor, targetActor) | ||
| 125 | |||
| 126 | return true | ||
| 127 | } | ||
| 128 | |||
| 129 | return false | ||
| 130 | } | ||
| 131 | |||
| 132 | async function acceptIfNeeded (actorFollow: MActorFollow, targetActor: MActorFull, transaction: Transaction) { | ||
| 133 | // Set the follow as accepted if the remote actor follows a channel or account | ||
| 134 | // Or if the instance automatically accepts followers | ||
| 135 | if (actorFollow.state === 'accepted') return | ||
| 136 | if (!await isFollowingInstance(targetActor)) return | ||
| 137 | if (CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL === true && await isFollowingInstance(targetActor)) return | ||
| 138 | |||
| 139 | actorFollow.state = 'accepted' | ||
| 140 | |||
| 141 | await actorFollow.save({ transaction }) | ||
| 142 | } | ||
| 143 | |||
| 144 | async function fixFollowURLIfNeeded (actorFollow: MActorFollow, activityId: string, transaction: Transaction) { | ||
| 145 | // Before PeerTube V3 we did not save the follow ID. Try to fix these old follows | ||
| 146 | if (!actorFollow.url) { | ||
| 147 | actorFollow.url = activityId | ||
| 148 | await actorFollow.save({ transaction }) | ||
| 149 | } | ||
| 150 | } | ||
| 151 | |||
| 152 | async function isFollowingInstance (targetActor: MActorId) { | ||
| 153 | const serverActor = await getServerActor() | ||
| 154 | |||
| 155 | return targetActor.id === serverActor.id | ||
| 156 | } | ||
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts deleted file mode 100644 index 580a05bcd..000000000 --- a/server/lib/activitypub/process/process-like.ts +++ /dev/null | |||
| @@ -1,60 +0,0 @@ | |||
| 1 | import { VideoModel } from '@server/models/video/video' | ||
| 2 | import { ActivityLike } from '../../../../shared/models/activitypub' | ||
| 3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 4 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 5 | import { getAPId } from '../../../lib/activitypub/activity' | ||
| 6 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
| 7 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 8 | import { MActorSignature } from '../../../types/models' | ||
| 9 | import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' | ||
| 10 | |||
| 11 | async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { | ||
| 12 | const { activity, byActor } = options | ||
| 13 | |||
| 14 | return retryTransactionWrapper(processLikeVideo, byActor, activity) | ||
| 15 | } | ||
| 16 | |||
| 17 | // --------------------------------------------------------------------------- | ||
| 18 | |||
| 19 | export { | ||
| 20 | processLikeActivity | ||
| 21 | } | ||
| 22 | |||
| 23 | // --------------------------------------------------------------------------- | ||
| 24 | |||
| 25 | async function processLikeVideo (byActor: MActorSignature, activity: ActivityLike) { | ||
| 26 | const videoUrl = getAPId(activity.object) | ||
| 27 | |||
| 28 | const byAccount = byActor.Account | ||
| 29 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | ||
| 30 | |||
| 31 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: videoUrl, fetchType: 'only-video' }) | ||
| 32 | |||
| 33 | // We don't care about likes of remote videos | ||
| 34 | if (!onlyVideo.isOwned()) return | ||
| 35 | |||
| 36 | return sequelizeTypescript.transaction(async t => { | ||
| 37 | const video = await VideoModel.loadFull(onlyVideo.id, t) | ||
| 38 | |||
| 39 | const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) | ||
| 40 | if (existingRate && existingRate.type === 'like') return | ||
| 41 | |||
| 42 | if (existingRate && existingRate.type === 'dislike') { | ||
| 43 | await video.decrement('dislikes', { transaction: t }) | ||
| 44 | video.dislikes-- | ||
| 45 | } | ||
| 46 | |||
| 47 | await video.increment('likes', { transaction: t }) | ||
| 48 | video.likes++ | ||
| 49 | |||
| 50 | const rate = existingRate || new AccountVideoRateModel() | ||
| 51 | rate.type = 'like' | ||
| 52 | rate.videoId = video.id | ||
| 53 | rate.accountId = byAccount.id | ||
| 54 | rate.url = activity.id | ||
| 55 | |||
| 56 | await rate.save({ transaction: t }) | ||
| 57 | |||
| 58 | await federateVideoIfNeeded(video, false, t) | ||
| 59 | }) | ||
| 60 | } | ||
diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts deleted file mode 100644 index db7ff24d8..000000000 --- a/server/lib/activitypub/process/process-reject.ts +++ /dev/null | |||
| @@ -1,33 +0,0 @@ | |||
| 1 | import { ActivityReject } from '../../../../shared/models/activitypub/activity' | ||
| 2 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 3 | import { ActorFollowModel } from '../../../models/actor/actor-follow' | ||
| 4 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 5 | import { MActor } from '../../../types/models' | ||
| 6 | |||
| 7 | async function processRejectActivity (options: APProcessorOptions<ActivityReject>) { | ||
| 8 | const { byActor: targetActor, inboxActor } = options | ||
| 9 | if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') | ||
| 10 | |||
| 11 | return processReject(inboxActor, targetActor) | ||
| 12 | } | ||
| 13 | |||
| 14 | // --------------------------------------------------------------------------- | ||
| 15 | |||
| 16 | export { | ||
| 17 | processRejectActivity | ||
| 18 | } | ||
| 19 | |||
| 20 | // --------------------------------------------------------------------------- | ||
| 21 | |||
| 22 | async function processReject (follower: MActor, targetActor: MActor) { | ||
| 23 | return sequelizeTypescript.transaction(async t => { | ||
| 24 | const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t) | ||
| 25 | |||
| 26 | if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`) | ||
| 27 | |||
| 28 | actorFollow.state = 'rejected' | ||
| 29 | await actorFollow.save({ transaction: t }) | ||
| 30 | |||
| 31 | return undefined | ||
| 32 | }) | ||
| 33 | } | ||
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts deleted file mode 100644 index a9d8199de..000000000 --- a/server/lib/activitypub/process/process-undo.ts +++ /dev/null | |||
| @@ -1,183 +0,0 @@ | |||
| 1 | import { VideoModel } from '@server/models/video/video' | ||
| 2 | import { | ||
| 3 | ActivityAnnounce, | ||
| 4 | ActivityCreate, | ||
| 5 | ActivityDislike, | ||
| 6 | ActivityFollow, | ||
| 7 | ActivityLike, | ||
| 8 | ActivityUndo, | ||
| 9 | ActivityUndoObject, | ||
| 10 | CacheFileObject | ||
| 11 | } from '../../../../shared/models/activitypub' | ||
| 12 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 13 | import { logger } from '../../../helpers/logger' | ||
| 14 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 15 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
| 16 | import { ActorModel } from '../../../models/actor/actor' | ||
| 17 | import { ActorFollowModel } from '../../../models/actor/actor-follow' | ||
| 18 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | ||
| 19 | import { VideoShareModel } from '../../../models/video/video-share' | ||
| 20 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 21 | import { MActorSignature } from '../../../types/models' | ||
| 22 | import { fetchAPObjectIfNeeded } from '../activity' | ||
| 23 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | ||
| 24 | import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' | ||
| 25 | |||
| 26 | async function processUndoActivity (options: APProcessorOptions<ActivityUndo<ActivityUndoObject>>) { | ||
| 27 | const { activity, byActor } = options | ||
| 28 | const activityToUndo = activity.object | ||
| 29 | |||
| 30 | if (activityToUndo.type === 'Like') { | ||
| 31 | return retryTransactionWrapper(processUndoLike, byActor, activity) | ||
| 32 | } | ||
| 33 | |||
| 34 | if (activityToUndo.type === 'Create') { | ||
| 35 | const objectToUndo = await fetchAPObjectIfNeeded<CacheFileObject>(activityToUndo.object) | ||
| 36 | |||
| 37 | if (objectToUndo.type === 'CacheFile') { | ||
| 38 | return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo) | ||
| 39 | } | ||
| 40 | } | ||
| 41 | |||
| 42 | if (activityToUndo.type === 'Dislike') { | ||
| 43 | return retryTransactionWrapper(processUndoDislike, byActor, activity) | ||
| 44 | } | ||
| 45 | |||
| 46 | if (activityToUndo.type === 'Follow') { | ||
| 47 | return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) | ||
| 48 | } | ||
| 49 | |||
| 50 | if (activityToUndo.type === 'Announce') { | ||
| 51 | return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo) | ||
| 52 | } | ||
| 53 | |||
| 54 | logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) | ||
| 55 | |||
| 56 | return undefined | ||
| 57 | } | ||
| 58 | |||
| 59 | // --------------------------------------------------------------------------- | ||
| 60 | |||
| 61 | export { | ||
| 62 | processUndoActivity | ||
| 63 | } | ||
| 64 | |||
| 65 | // --------------------------------------------------------------------------- | ||
| 66 | |||
| 67 | async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo<ActivityLike>) { | ||
| 68 | const likeActivity = activity.object | ||
| 69 | |||
| 70 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) | ||
| 71 | // We don't care about likes of remote videos | ||
| 72 | if (!onlyVideo.isOwned()) return | ||
| 73 | |||
| 74 | return sequelizeTypescript.transaction(async t => { | ||
| 75 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | ||
| 76 | |||
| 77 | const video = await VideoModel.loadFull(onlyVideo.id, t) | ||
| 78 | const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, likeActivity.id, t) | ||
| 79 | if (!rate || rate.type !== 'like') { | ||
| 80 | logger.warn('Unknown like by account %d for video %d.', byActor.Account.id, video.id) | ||
| 81 | return | ||
| 82 | } | ||
| 83 | |||
| 84 | await rate.destroy({ transaction: t }) | ||
| 85 | await video.decrement('likes', { transaction: t }) | ||
| 86 | |||
| 87 | video.likes-- | ||
| 88 | await federateVideoIfNeeded(video, false, t) | ||
| 89 | }) | ||
| 90 | } | ||
| 91 | |||
| 92 | async function processUndoDislike (byActor: MActorSignature, activity: ActivityUndo<ActivityDislike>) { | ||
| 93 | const dislikeActivity = activity.object | ||
| 94 | |||
| 95 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeActivity.object }) | ||
| 96 | // We don't care about likes of remote videos | ||
| 97 | if (!onlyVideo.isOwned()) return | ||
| 98 | |||
| 99 | return sequelizeTypescript.transaction(async t => { | ||
| 100 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | ||
| 101 | |||
| 102 | const video = await VideoModel.loadFull(onlyVideo.id, t) | ||
| 103 | const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, dislikeActivity.id, t) | ||
| 104 | if (!rate || rate.type !== 'dislike') { | ||
| 105 | logger.warn(`Unknown dislike by account %d for video %d.`, byActor.Account.id, video.id) | ||
| 106 | return | ||
| 107 | } | ||
| 108 | |||
| 109 | await rate.destroy({ transaction: t }) | ||
| 110 | await video.decrement('dislikes', { transaction: t }) | ||
| 111 | video.dislikes-- | ||
| 112 | |||
| 113 | await federateVideoIfNeeded(video, false, t) | ||
| 114 | }) | ||
| 115 | } | ||
| 116 | |||
| 117 | // --------------------------------------------------------------------------- | ||
| 118 | |||
| 119 | async function processUndoCacheFile ( | ||
| 120 | byActor: MActorSignature, | ||
| 121 | activity: ActivityUndo<ActivityCreate<CacheFileObject>>, | ||
| 122 | cacheFileObject: CacheFileObject | ||
| 123 | ) { | ||
| 124 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) | ||
| 125 | |||
| 126 | return sequelizeTypescript.transaction(async t => { | ||
| 127 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) | ||
| 128 | if (!cacheFile) { | ||
| 129 | logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) | ||
| 130 | return | ||
| 131 | } | ||
| 132 | |||
| 133 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') | ||
| 134 | |||
| 135 | await cacheFile.destroy({ transaction: t }) | ||
| 136 | |||
| 137 | if (video.isOwned()) { | ||
| 138 | // Don't resend the activity to the sender | ||
| 139 | const exceptions = [ byActor ] | ||
| 140 | |||
| 141 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | ||
| 142 | } | ||
| 143 | }) | ||
| 144 | } | ||
| 145 | |||
| 146 | function processUndoAnnounce (byActor: MActorSignature, announceActivity: ActivityAnnounce) { | ||
| 147 | return sequelizeTypescript.transaction(async t => { | ||
| 148 | const share = await VideoShareModel.loadByUrl(announceActivity.id, t) | ||
| 149 | if (!share) { | ||
| 150 | logger.warn('Unknown video share %d', announceActivity.id) | ||
| 151 | return | ||
| 152 | } | ||
| 153 | |||
| 154 | if (share.actorId !== byActor.id) throw new Error(`${share.url} is not shared by ${byActor.url}.`) | ||
| 155 | |||
| 156 | await share.destroy({ transaction: t }) | ||
| 157 | |||
| 158 | if (share.Video.isOwned()) { | ||
| 159 | // Don't resend the activity to the sender | ||
| 160 | const exceptions = [ byActor ] | ||
| 161 | |||
| 162 | await forwardVideoRelatedActivity(announceActivity, t, exceptions, share.Video) | ||
| 163 | } | ||
| 164 | }) | ||
| 165 | } | ||
| 166 | |||
| 167 | // --------------------------------------------------------------------------- | ||
| 168 | |||
| 169 | function processUndoFollow (follower: MActorSignature, followActivity: ActivityFollow) { | ||
| 170 | return sequelizeTypescript.transaction(async t => { | ||
| 171 | const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t) | ||
| 172 | const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) | ||
| 173 | |||
| 174 | if (!actorFollow) { | ||
| 175 | logger.warn('Unknown actor follow %d -> %d.', follower.id, following.id) | ||
| 176 | return | ||
| 177 | } | ||
| 178 | |||
| 179 | await actorFollow.destroy({ transaction: t }) | ||
| 180 | |||
| 181 | return undefined | ||
| 182 | }) | ||
| 183 | } | ||
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts deleted file mode 100644 index 304ed9de6..000000000 --- a/server/lib/activitypub/process/process-update.ts +++ /dev/null | |||
| @@ -1,119 +0,0 @@ | |||
| 1 | import { isRedundancyAccepted } from '@server/lib/redundancy' | ||
| 2 | import { ActivityUpdate, ActivityUpdateObject, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' | ||
| 3 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' | ||
| 4 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' | ||
| 5 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | ||
| 6 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' | ||
| 7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 8 | import { logger } from '../../../helpers/logger' | ||
| 9 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 10 | import { ActorModel } from '../../../models/actor/actor' | ||
| 11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 12 | import { MActorFull, MActorSignature } from '../../../types/models' | ||
| 13 | import { fetchAPObjectIfNeeded } from '../activity' | ||
| 14 | import { APActorUpdater } from '../actors/updater' | ||
| 15 | import { createOrUpdateCacheFile } from '../cache-file' | ||
| 16 | import { createOrUpdateVideoPlaylist } from '../playlists' | ||
| 17 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | ||
| 18 | import { APVideoUpdater, getOrCreateAPVideo } from '../videos' | ||
| 19 | |||
| 20 | async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate<ActivityUpdateObject>>) { | ||
| 21 | const { activity, byActor } = options | ||
| 22 | |||
| 23 | const object = await fetchAPObjectIfNeeded(activity.object) | ||
| 24 | const objectType = object.type | ||
| 25 | |||
| 26 | if (objectType === 'Video') { | ||
| 27 | return retryTransactionWrapper(processUpdateVideo, activity) | ||
| 28 | } | ||
| 29 | |||
| 30 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { | ||
| 31 | // We need more attributes | ||
| 32 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) | ||
| 33 | return retryTransactionWrapper(processUpdateActor, byActorFull, object) | ||
| 34 | } | ||
| 35 | |||
| 36 | if (objectType === 'CacheFile') { | ||
| 37 | // We need more attributes | ||
| 38 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) | ||
| 39 | return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity, object) | ||
| 40 | } | ||
| 41 | |||
| 42 | if (objectType === 'Playlist') { | ||
| 43 | return retryTransactionWrapper(processUpdatePlaylist, byActor, activity, object) | ||
| 44 | } | ||
| 45 | |||
| 46 | return undefined | ||
| 47 | } | ||
| 48 | |||
| 49 | // --------------------------------------------------------------------------- | ||
| 50 | |||
| 51 | export { | ||
| 52 | processUpdateActivity | ||
| 53 | } | ||
| 54 | |||
| 55 | // --------------------------------------------------------------------------- | ||
| 56 | |||
| 57 | async function processUpdateVideo (activity: ActivityUpdate<VideoObject | string>) { | ||
| 58 | const videoObject = activity.object as VideoObject | ||
| 59 | |||
| 60 | if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { | ||
| 61 | logger.debug('Video sent by update is not valid.', { videoObject }) | ||
| 62 | return undefined | ||
| 63 | } | ||
| 64 | |||
| 65 | const { video, created } = await getOrCreateAPVideo({ | ||
| 66 | videoObject: videoObject.id, | ||
| 67 | allowRefresh: false, | ||
| 68 | fetchType: 'all' | ||
| 69 | }) | ||
| 70 | // We did not have this video, it has been created so no need to update | ||
| 71 | if (created) return | ||
| 72 | |||
| 73 | const updater = new APVideoUpdater(videoObject, video) | ||
| 74 | return updater.update(activity.to) | ||
| 75 | } | ||
| 76 | |||
| 77 | async function processUpdateCacheFile ( | ||
| 78 | byActor: MActorSignature, | ||
| 79 | activity: ActivityUpdate<CacheFileObject | string>, | ||
| 80 | cacheFileObject: CacheFileObject | ||
| 81 | ) { | ||
| 82 | if (await isRedundancyAccepted(activity, byActor) !== true) return | ||
| 83 | |||
| 84 | if (!isCacheFileObjectValid(cacheFileObject)) { | ||
| 85 | logger.debug('Cache file object sent by update is not valid.', { cacheFileObject }) | ||
| 86 | return undefined | ||
| 87 | } | ||
| 88 | |||
| 89 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) | ||
| 90 | |||
| 91 | await sequelizeTypescript.transaction(async t => { | ||
| 92 | await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) | ||
| 93 | }) | ||
| 94 | |||
| 95 | if (video.isOwned()) { | ||
| 96 | // Don't resend the activity to the sender | ||
| 97 | const exceptions = [ byActor ] | ||
| 98 | |||
| 99 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
| 100 | } | ||
| 101 | } | ||
| 102 | |||
| 103 | async function processUpdateActor (actor: MActorFull, actorObject: ActivityPubActor) { | ||
| 104 | logger.debug('Updating remote account "%s".', actorObject.url) | ||
| 105 | |||
| 106 | const updater = new APActorUpdater(actorObject, actor) | ||
| 107 | return updater.update() | ||
| 108 | } | ||
| 109 | |||
| 110 | async function processUpdatePlaylist ( | ||
| 111 | byActor: MActorSignature, | ||
| 112 | activity: ActivityUpdate<PlaylistObject | string>, | ||
| 113 | playlistObject: PlaylistObject | ||
| 114 | ) { | ||
| 115 | const byAccount = byActor.Account | ||
| 116 | if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) | ||
| 117 | |||
| 118 | await createOrUpdateVideoPlaylist(playlistObject, activity.to) | ||
| 119 | } | ||
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts deleted file mode 100644 index e49506d82..000000000 --- a/server/lib/activitypub/process/process-view.ts +++ /dev/null | |||
| @@ -1,42 +0,0 @@ | |||
| 1 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' | ||
| 2 | import { ActivityView } from '../../../../shared/models/activitypub' | ||
| 3 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 4 | import { MActorSignature } from '../../../types/models' | ||
| 5 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | ||
| 6 | import { getOrCreateAPVideo } from '../videos' | ||
| 7 | |||
| 8 | async function processViewActivity (options: APProcessorOptions<ActivityView>) { | ||
| 9 | const { activity, byActor } = options | ||
| 10 | |||
| 11 | return processCreateView(activity, byActor) | ||
| 12 | } | ||
| 13 | |||
| 14 | // --------------------------------------------------------------------------- | ||
| 15 | |||
| 16 | export { | ||
| 17 | processViewActivity | ||
| 18 | } | ||
| 19 | |||
| 20 | // --------------------------------------------------------------------------- | ||
| 21 | |||
| 22 | async function processCreateView (activity: ActivityView, byActor: MActorSignature) { | ||
| 23 | const videoObject = activity.object | ||
| 24 | |||
| 25 | const { video } = await getOrCreateAPVideo({ | ||
| 26 | videoObject, | ||
| 27 | fetchType: 'only-video', | ||
| 28 | allowRefresh: false | ||
| 29 | }) | ||
| 30 | |||
| 31 | const viewerExpires = activity.expires | ||
| 32 | ? new Date(activity.expires) | ||
| 33 | : undefined | ||
| 34 | |||
| 35 | await VideoViewsManager.Instance.processRemoteView({ video, viewerId: activity.id, viewerExpires }) | ||
| 36 | |||
| 37 | if (video.isOwned()) { | ||
| 38 | // Forward the view but don't resend the activity to the sender | ||
| 39 | const exceptions = [ byActor ] | ||
| 40 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
| 41 | } | ||
| 42 | } | ||
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts deleted file mode 100644 index 2bc3dce03..000000000 --- a/server/lib/activitypub/process/process.ts +++ /dev/null | |||
| @@ -1,92 +0,0 @@ | |||
| 1 | import { StatsManager } from '@server/lib/stat-manager' | ||
| 2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | ||
| 5 | import { MActorDefault, MActorSignature } from '../../../types/models' | ||
| 6 | import { getAPId } from '../activity' | ||
| 7 | import { getOrCreateAPActor } from '../actors' | ||
| 8 | import { checkUrlsSameHost } from '../url' | ||
| 9 | import { processAcceptActivity } from './process-accept' | ||
| 10 | import { processAnnounceActivity } from './process-announce' | ||
| 11 | import { processCreateActivity } from './process-create' | ||
| 12 | import { processDeleteActivity } from './process-delete' | ||
| 13 | import { processDislikeActivity } from './process-dislike' | ||
| 14 | import { processFlagActivity } from './process-flag' | ||
| 15 | import { processFollowActivity } from './process-follow' | ||
| 16 | import { processLikeActivity } from './process-like' | ||
| 17 | import { processRejectActivity } from './process-reject' | ||
| 18 | import { processUndoActivity } from './process-undo' | ||
| 19 | import { processUpdateActivity } from './process-update' | ||
| 20 | import { processViewActivity } from './process-view' | ||
| 21 | |||
| 22 | const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions<Activity>) => Promise<any> } = { | ||
| 23 | Create: processCreateActivity, | ||
| 24 | Update: processUpdateActivity, | ||
| 25 | Delete: processDeleteActivity, | ||
| 26 | Follow: processFollowActivity, | ||
| 27 | Accept: processAcceptActivity, | ||
| 28 | Reject: processRejectActivity, | ||
| 29 | Announce: processAnnounceActivity, | ||
| 30 | Undo: processUndoActivity, | ||
| 31 | Like: processLikeActivity, | ||
| 32 | Dislike: processDislikeActivity, | ||
| 33 | Flag: processFlagActivity, | ||
| 34 | View: processViewActivity | ||
| 35 | } | ||
| 36 | |||
| 37 | async function processActivities ( | ||
| 38 | activities: Activity[], | ||
| 39 | options: { | ||
| 40 | signatureActor?: MActorSignature | ||
| 41 | inboxActor?: MActorDefault | ||
| 42 | outboxUrl?: string | ||
| 43 | fromFetch?: boolean | ||
| 44 | } = {} | ||
| 45 | ) { | ||
| 46 | const { outboxUrl, signatureActor, inboxActor, fromFetch = false } = options | ||
| 47 | |||
| 48 | const actorsCache: { [ url: string ]: MActorSignature } = {} | ||
| 49 | |||
| 50 | for (const activity of activities) { | ||
| 51 | if (!signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { | ||
| 52 | logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) | ||
| 53 | continue | ||
| 54 | } | ||
| 55 | |||
| 56 | const actorUrl = getAPId(activity.actor) | ||
| 57 | |||
| 58 | // When we fetch remote data, we don't have signature | ||
| 59 | if (signatureActor && actorUrl !== signatureActor.url) { | ||
| 60 | logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, signatureActor.url) | ||
| 61 | continue | ||
| 62 | } | ||
| 63 | |||
| 64 | if (outboxUrl && checkUrlsSameHost(outboxUrl, actorUrl) !== true) { | ||
| 65 | logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', outboxUrl, actorUrl) | ||
| 66 | continue | ||
| 67 | } | ||
| 68 | |||
| 69 | const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl) | ||
| 70 | actorsCache[actorUrl] = byActor | ||
| 71 | |||
| 72 | const activityProcessor = processActivity[activity.type] | ||
| 73 | if (activityProcessor === undefined) { | ||
| 74 | logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) | ||
| 75 | continue | ||
| 76 | } | ||
| 77 | |||
| 78 | try { | ||
| 79 | await activityProcessor({ activity, byActor, inboxActor, fromFetch }) | ||
| 80 | |||
| 81 | StatsManager.Instance.addInboxProcessedSuccess(activity.type) | ||
| 82 | } catch (err) { | ||
| 83 | logger.warn('Cannot process activity %s.', activity.type, { err }) | ||
| 84 | |||
| 85 | StatsManager.Instance.addInboxProcessedError(activity.type) | ||
| 86 | } | ||
| 87 | } | ||
| 88 | } | ||
| 89 | |||
| 90 | export { | ||
| 91 | processActivities | ||
| 92 | } | ||
diff --git a/server/lib/activitypub/send/http.ts b/server/lib/activitypub/send/http.ts deleted file mode 100644 index b461aa55d..000000000 --- a/server/lib/activitypub/send/http.ts +++ /dev/null | |||
| @@ -1,73 +0,0 @@ | |||
| 1 | import { buildDigest, signJsonLDObject } from '@server/helpers/peertube-crypto' | ||
| 2 | import { ACTIVITY_PUB, HTTP_SIGNATURE } from '@server/initializers/constants' | ||
| 3 | import { ActorModel } from '@server/models/actor/actor' | ||
| 4 | import { getServerActor } from '@server/models/application/application' | ||
| 5 | import { MActor } from '@server/types/models' | ||
| 6 | import { ContextType } from '@shared/models/activitypub/context' | ||
| 7 | import { activityPubContextify } from '../context' | ||
| 8 | |||
| 9 | type Payload <T> = { body: T, contextType: ContextType, signatureActorId?: number } | ||
| 10 | |||
| 11 | async function computeBody <T> ( | ||
| 12 | payload: Payload<T> | ||
| 13 | ): Promise<T | T & { type: 'RsaSignature2017', creator: string, created: string }> { | ||
| 14 | let body = payload.body | ||
| 15 | |||
| 16 | if (payload.signatureActorId) { | ||
| 17 | const actorSignature = await ActorModel.load(payload.signatureActorId) | ||
| 18 | if (!actorSignature) throw new Error('Unknown signature actor id.') | ||
| 19 | |||
| 20 | body = await signAndContextify(actorSignature, payload.body, payload.contextType) | ||
| 21 | } | ||
| 22 | |||
| 23 | return body | ||
| 24 | } | ||
| 25 | |||
| 26 | async function buildSignedRequestOptions (options: { | ||
| 27 | signatureActorId?: number | ||
| 28 | hasPayload: boolean | ||
| 29 | }) { | ||
| 30 | let actor: MActor | null | ||
| 31 | |||
| 32 | if (options.signatureActorId) { | ||
| 33 | actor = await ActorModel.load(options.signatureActorId) | ||
| 34 | if (!actor) throw new Error('Unknown signature actor id.') | ||
| 35 | } else { | ||
| 36 | // We need to sign the request, so use the server | ||
| 37 | actor = await getServerActor() | ||
| 38 | } | ||
| 39 | |||
| 40 | const keyId = actor.url | ||
| 41 | return { | ||
| 42 | algorithm: HTTP_SIGNATURE.ALGORITHM, | ||
| 43 | authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, | ||
| 44 | keyId, | ||
| 45 | key: actor.privateKey, | ||
| 46 | headers: options.hasPayload | ||
| 47 | ? HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD | ||
| 48 | : HTTP_SIGNATURE.HEADERS_TO_SIGN_WITHOUT_PAYLOAD | ||
| 49 | } | ||
| 50 | } | ||
| 51 | |||
| 52 | function buildGlobalHeaders (body: any) { | ||
| 53 | return { | ||
| 54 | 'digest': buildDigest(body), | ||
| 55 | 'content-type': 'application/activity+json', | ||
| 56 | 'accept': ACTIVITY_PUB.ACCEPT_HEADER | ||
| 57 | } | ||
| 58 | } | ||
| 59 | |||
| 60 | async function signAndContextify <T> (byActor: MActor, data: T, contextType: ContextType | null) { | ||
| 61 | const activity = contextType | ||
| 62 | ? await activityPubContextify(data, contextType) | ||
| 63 | : data | ||
| 64 | |||
| 65 | return signJsonLDObject(byActor, activity) | ||
| 66 | } | ||
| 67 | |||
| 68 | export { | ||
| 69 | buildGlobalHeaders, | ||
| 70 | computeBody, | ||
| 71 | buildSignedRequestOptions, | ||
| 72 | signAndContextify | ||
| 73 | } | ||
diff --git a/server/lib/activitypub/send/index.ts b/server/lib/activitypub/send/index.ts deleted file mode 100644 index 852ea2e74..000000000 --- a/server/lib/activitypub/send/index.ts +++ /dev/null | |||
| @@ -1,10 +0,0 @@ | |||
| 1 | export * from './http' | ||
| 2 | export * from './send-accept' | ||
| 3 | export * from './send-announce' | ||
| 4 | export * from './send-create' | ||
| 5 | export * from './send-delete' | ||
| 6 | export * from './send-follow' | ||
| 7 | export * from './send-like' | ||
| 8 | export * from './send-reject' | ||
| 9 | export * from './send-undo' | ||
| 10 | export * from './send-update' | ||
diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts deleted file mode 100644 index 4c9bcbb0b..000000000 --- a/server/lib/activitypub/send/send-accept.ts +++ /dev/null | |||
| @@ -1,47 +0,0 @@ | |||
| 1 | import { ActivityAccept, ActivityFollow } from '@shared/models' | ||
| 2 | import { logger } from '../../../helpers/logger' | ||
| 3 | import { MActor, MActorFollowActors } from '../../../types/models' | ||
| 4 | import { getLocalActorFollowAcceptActivityPubUrl } from '../url' | ||
| 5 | import { buildFollowActivity } from './send-follow' | ||
| 6 | import { unicastTo } from './shared/send-utils' | ||
| 7 | |||
| 8 | function sendAccept (actorFollow: MActorFollowActors) { | ||
| 9 | const follower = actorFollow.ActorFollower | ||
| 10 | const me = actorFollow.ActorFollowing | ||
| 11 | |||
| 12 | if (!follower.serverId) { // This should never happen | ||
| 13 | logger.warn('Do not sending accept to local follower.') | ||
| 14 | return | ||
| 15 | } | ||
| 16 | |||
| 17 | logger.info('Creating job to accept follower %s.', follower.url) | ||
| 18 | |||
| 19 | const followData = buildFollowActivity(actorFollow.url, follower, me) | ||
| 20 | |||
| 21 | const url = getLocalActorFollowAcceptActivityPubUrl(actorFollow) | ||
| 22 | const data = buildAcceptActivity(url, me, followData) | ||
| 23 | |||
| 24 | return unicastTo({ | ||
| 25 | data, | ||
| 26 | byActor: me, | ||
| 27 | toActorUrl: follower.inboxUrl, | ||
| 28 | contextType: 'Accept' | ||
| 29 | }) | ||
| 30 | } | ||
| 31 | |||
| 32 | // --------------------------------------------------------------------------- | ||
| 33 | |||
| 34 | export { | ||
| 35 | sendAccept | ||
| 36 | } | ||
| 37 | |||
| 38 | // --------------------------------------------------------------------------- | ||
| 39 | |||
| 40 | function buildAcceptActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityAccept { | ||
| 41 | return { | ||
| 42 | type: 'Accept', | ||
| 43 | id: url, | ||
| 44 | actor: byActor.url, | ||
| 45 | object: followActivityData | ||
| 46 | } | ||
| 47 | } | ||
diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts deleted file mode 100644 index 6c078b047..000000000 --- a/server/lib/activitypub/send/send-announce.ts +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ActivityAnnounce, ActivityAudience } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { MActorLight, MVideo } from '../../../types/models' | ||
| 5 | import { MVideoShare } from '../../../types/models/video' | ||
| 6 | import { audiencify, getAudience } from '../audience' | ||
| 7 | import { getActorsInvolvedInVideo, getAudienceFromFollowersOf } from './shared' | ||
| 8 | import { broadcastToFollowers } from './shared/send-utils' | ||
| 9 | |||
| 10 | async function buildAnnounceWithVideoAudience ( | ||
| 11 | byActor: MActorLight, | ||
| 12 | videoShare: MVideoShare, | ||
| 13 | video: MVideo, | ||
| 14 | t: Transaction | ||
| 15 | ) { | ||
| 16 | const announcedObject = video.url | ||
| 17 | |||
| 18 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | ||
| 19 | const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) | ||
| 20 | |||
| 21 | const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) | ||
| 22 | |||
| 23 | return { activity, actorsInvolvedInVideo } | ||
| 24 | } | ||
| 25 | |||
| 26 | async function sendVideoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { | ||
| 27 | const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) | ||
| 28 | |||
| 29 | logger.info('Creating job to send announce %s.', videoShare.url) | ||
| 30 | |||
| 31 | return broadcastToFollowers({ | ||
| 32 | data: activity, | ||
| 33 | byActor, | ||
| 34 | toFollowersOf: actorsInvolvedInVideo, | ||
| 35 | transaction, | ||
| 36 | actorsException: [ byActor ], | ||
| 37 | contextType: 'Announce' | ||
| 38 | }) | ||
| 39 | } | ||
| 40 | |||
| 41 | function buildAnnounceActivity (url: string, byActor: MActorLight, object: string, audience?: ActivityAudience): ActivityAnnounce { | ||
| 42 | if (!audience) audience = getAudience(byActor) | ||
| 43 | |||
| 44 | return audiencify({ | ||
| 45 | type: 'Announce' as 'Announce', | ||
| 46 | id: url, | ||
| 47 | actor: byActor.url, | ||
| 48 | object | ||
| 49 | }, audience) | ||
| 50 | } | ||
| 51 | |||
| 52 | // --------------------------------------------------------------------------- | ||
| 53 | |||
| 54 | export { | ||
| 55 | sendVideoAnnounce, | ||
| 56 | buildAnnounceActivity, | ||
| 57 | buildAnnounceWithVideoAudience | ||
| 58 | } | ||
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts deleted file mode 100644 index 2cd4db14d..000000000 --- a/server/lib/activitypub/send/send-create.ts +++ /dev/null | |||
| @@ -1,226 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { getServerActor } from '@server/models/application/application' | ||
| 3 | import { | ||
| 4 | ActivityAudience, | ||
| 5 | ActivityCreate, | ||
| 6 | ActivityCreateObject, | ||
| 7 | ContextType, | ||
| 8 | VideoCommentObject, | ||
| 9 | VideoPlaylistPrivacy, | ||
| 10 | VideoPrivacy | ||
| 11 | } from '@shared/models' | ||
| 12 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | ||
| 13 | import { VideoCommentModel } from '../../../models/video/video-comment' | ||
| 14 | import { | ||
| 15 | MActorLight, | ||
| 16 | MCommentOwnerVideo, | ||
| 17 | MLocalVideoViewerWithWatchSections, | ||
| 18 | MVideoAccountLight, | ||
| 19 | MVideoAP, | ||
| 20 | MVideoPlaylistFull, | ||
| 21 | MVideoRedundancyFileVideo, | ||
| 22 | MVideoRedundancyStreamingPlaylistVideo | ||
| 23 | } from '../../../types/models' | ||
| 24 | import { audiencify, getAudience } from '../audience' | ||
| 25 | import { | ||
| 26 | broadcastToActors, | ||
| 27 | broadcastToFollowers, | ||
| 28 | getActorsInvolvedInVideo, | ||
| 29 | getAudienceFromFollowersOf, | ||
| 30 | getVideoCommentAudience, | ||
| 31 | sendVideoActivityToOrigin, | ||
| 32 | sendVideoRelatedActivity, | ||
| 33 | unicastTo | ||
| 34 | } from './shared' | ||
| 35 | |||
| 36 | const lTags = loggerTagsFactory('ap', 'create') | ||
| 37 | |||
| 38 | async function sendCreateVideo (video: MVideoAP, transaction: Transaction) { | ||
| 39 | if (!video.hasPrivacyForFederation()) return undefined | ||
| 40 | |||
| 41 | logger.info('Creating job to send video creation of %s.', video.url, lTags(video.uuid)) | ||
| 42 | |||
| 43 | const byActor = video.VideoChannel.Account.Actor | ||
| 44 | const videoObject = await video.toActivityPubObject() | ||
| 45 | |||
| 46 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) | ||
| 47 | const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience) | ||
| 48 | |||
| 49 | return broadcastToFollowers({ | ||
| 50 | data: createActivity, | ||
| 51 | byActor, | ||
| 52 | toFollowersOf: [ byActor ], | ||
| 53 | transaction, | ||
| 54 | contextType: 'Video' | ||
| 55 | }) | ||
| 56 | } | ||
| 57 | |||
| 58 | async function sendCreateCacheFile ( | ||
| 59 | byActor: MActorLight, | ||
| 60 | video: MVideoAccountLight, | ||
| 61 | fileRedundancy: MVideoRedundancyStreamingPlaylistVideo | MVideoRedundancyFileVideo | ||
| 62 | ) { | ||
| 63 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url, lTags(video.uuid)) | ||
| 64 | |||
| 65 | return sendVideoRelatedCreateActivity({ | ||
| 66 | byActor, | ||
| 67 | video, | ||
| 68 | url: fileRedundancy.url, | ||
| 69 | object: fileRedundancy.toActivityPubObject(), | ||
| 70 | contextType: 'CacheFile' | ||
| 71 | }) | ||
| 72 | } | ||
| 73 | |||
| 74 | async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) { | ||
| 75 | logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid)) | ||
| 76 | |||
| 77 | const byActor = await getServerActor() | ||
| 78 | |||
| 79 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 80 | return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience) | ||
| 81 | } | ||
| 82 | |||
| 83 | return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' }) | ||
| 84 | } | ||
| 85 | |||
| 86 | async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) { | ||
| 87 | if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined | ||
| 88 | |||
| 89 | logger.info('Creating job to send create video playlist of %s.', playlist.url, lTags(playlist.uuid)) | ||
| 90 | |||
| 91 | const byActor = playlist.OwnerAccount.Actor | ||
| 92 | const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) | ||
| 93 | |||
| 94 | const object = await playlist.toActivityPubObject(null, transaction) | ||
| 95 | const createActivity = buildCreateActivity(playlist.url, byActor, object, audience) | ||
| 96 | |||
| 97 | const serverActor = await getServerActor() | ||
| 98 | const toFollowersOf = [ byActor, serverActor ] | ||
| 99 | |||
| 100 | if (playlist.VideoChannel) toFollowersOf.push(playlist.VideoChannel.Actor) | ||
| 101 | |||
| 102 | return broadcastToFollowers({ | ||
| 103 | data: createActivity, | ||
| 104 | byActor, | ||
| 105 | toFollowersOf, | ||
| 106 | transaction, | ||
| 107 | contextType: 'Playlist' | ||
| 108 | }) | ||
| 109 | } | ||
| 110 | |||
| 111 | async function sendCreateVideoComment (comment: MCommentOwnerVideo, transaction: Transaction) { | ||
| 112 | logger.info('Creating job to send comment %s.', comment.url) | ||
| 113 | |||
| 114 | const isOrigin = comment.Video.isOwned() | ||
| 115 | |||
| 116 | const byActor = comment.Account.Actor | ||
| 117 | const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, transaction) | ||
| 118 | const commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject | ||
| 119 | |||
| 120 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, transaction) | ||
| 121 | // Add the actor that commented too | ||
| 122 | actorsInvolvedInComment.push(byActor) | ||
| 123 | |||
| 124 | const parentsCommentActors = threadParentComments.filter(c => !c.isDeleted()) | ||
| 125 | .map(c => c.Account.Actor) | ||
| 126 | |||
| 127 | let audience: ActivityAudience | ||
| 128 | if (isOrigin) { | ||
| 129 | audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) | ||
| 130 | } else { | ||
| 131 | audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors)) | ||
| 132 | } | ||
| 133 | |||
| 134 | const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) | ||
| 135 | |||
| 136 | // This was a reply, send it to the parent actors | ||
| 137 | const actorsException = [ byActor ] | ||
| 138 | await broadcastToActors({ | ||
| 139 | data: createActivity, | ||
| 140 | byActor, | ||
| 141 | toActors: parentsCommentActors, | ||
| 142 | transaction, | ||
| 143 | actorsException, | ||
| 144 | contextType: 'Comment' | ||
| 145 | }) | ||
| 146 | |||
| 147 | // Broadcast to our followers | ||
| 148 | await broadcastToFollowers({ | ||
| 149 | data: createActivity, | ||
| 150 | byActor, | ||
| 151 | toFollowersOf: [ byActor ], | ||
| 152 | transaction, | ||
| 153 | contextType: 'Comment' | ||
| 154 | }) | ||
| 155 | |||
| 156 | // Send to actors involved in the comment | ||
| 157 | if (isOrigin) { | ||
| 158 | return broadcastToFollowers({ | ||
| 159 | data: createActivity, | ||
| 160 | byActor, | ||
| 161 | toFollowersOf: actorsInvolvedInComment, | ||
| 162 | transaction, | ||
| 163 | actorsException, | ||
| 164 | contextType: 'Comment' | ||
| 165 | }) | ||
| 166 | } | ||
| 167 | |||
| 168 | // Send to origin | ||
| 169 | return transaction.afterCommit(() => { | ||
| 170 | return unicastTo({ | ||
| 171 | data: createActivity, | ||
| 172 | byActor, | ||
| 173 | toActorUrl: comment.Video.VideoChannel.Account.Actor.getSharedInbox(), | ||
| 174 | contextType: 'Comment' | ||
| 175 | }) | ||
| 176 | }) | ||
| 177 | } | ||
| 178 | |||
| 179 | function buildCreateActivity <T extends ActivityCreateObject> ( | ||
| 180 | url: string, | ||
| 181 | byActor: MActorLight, | ||
| 182 | object: T, | ||
| 183 | audience?: ActivityAudience | ||
| 184 | ): ActivityCreate<T> { | ||
| 185 | if (!audience) audience = getAudience(byActor) | ||
| 186 | |||
| 187 | return audiencify( | ||
| 188 | { | ||
| 189 | type: 'Create' as 'Create', | ||
| 190 | id: url + '/activity', | ||
| 191 | actor: byActor.url, | ||
| 192 | object: typeof object === 'string' | ||
| 193 | ? object | ||
| 194 | : audiencify(object, audience) | ||
| 195 | }, | ||
| 196 | audience | ||
| 197 | ) | ||
| 198 | } | ||
| 199 | |||
| 200 | // --------------------------------------------------------------------------- | ||
| 201 | |||
| 202 | export { | ||
| 203 | sendCreateVideo, | ||
| 204 | buildCreateActivity, | ||
| 205 | sendCreateVideoComment, | ||
| 206 | sendCreateVideoPlaylist, | ||
| 207 | sendCreateCacheFile, | ||
| 208 | sendCreateWatchAction | ||
| 209 | } | ||
| 210 | |||
| 211 | // --------------------------------------------------------------------------- | ||
| 212 | |||
| 213 | async function sendVideoRelatedCreateActivity (options: { | ||
| 214 | byActor: MActorLight | ||
| 215 | video: MVideoAccountLight | ||
| 216 | url: string | ||
| 217 | object: any | ||
| 218 | contextType: ContextType | ||
| 219 | transaction?: Transaction | ||
| 220 | }) { | ||
| 221 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 222 | return buildCreateActivity(options.url, options.byActor, options.object, audience) | ||
| 223 | } | ||
| 224 | |||
| 225 | return sendVideoRelatedActivity(activityBuilder, options) | ||
| 226 | } | ||
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts deleted file mode 100644 index 0d85d9001..000000000 --- a/server/lib/activitypub/send/send-delete.ts +++ /dev/null | |||
| @@ -1,158 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { getServerActor } from '@server/models/application/application' | ||
| 3 | import { ActivityAudience, ActivityDelete } from '@shared/models' | ||
| 4 | import { logger } from '../../../helpers/logger' | ||
| 5 | import { ActorModel } from '../../../models/actor/actor' | ||
| 6 | import { VideoCommentModel } from '../../../models/video/video-comment' | ||
| 7 | import { VideoShareModel } from '../../../models/video/video-share' | ||
| 8 | import { MActorUrl } from '../../../types/models' | ||
| 9 | import { MCommentOwnerVideo, MVideoAccountLight, MVideoPlaylistFullSummary } from '../../../types/models/video' | ||
| 10 | import { audiencify } from '../audience' | ||
| 11 | import { getDeleteActivityPubUrl } from '../url' | ||
| 12 | import { getActorsInvolvedInVideo, getVideoCommentAudience } from './shared' | ||
| 13 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './shared/send-utils' | ||
| 14 | |||
| 15 | async function sendDeleteVideo (video: MVideoAccountLight, transaction: Transaction) { | ||
| 16 | logger.info('Creating job to broadcast delete of video %s.', video.url) | ||
| 17 | |||
| 18 | const byActor = video.VideoChannel.Account.Actor | ||
| 19 | |||
| 20 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 21 | const url = getDeleteActivityPubUrl(video.url) | ||
| 22 | |||
| 23 | return buildDeleteActivity(url, video.url, byActor, audience) | ||
| 24 | } | ||
| 25 | |||
| 26 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'Delete', transaction }) | ||
| 27 | } | ||
| 28 | |||
| 29 | async function sendDeleteActor (byActor: ActorModel, transaction: Transaction) { | ||
| 30 | logger.info('Creating job to broadcast delete of actor %s.', byActor.url) | ||
| 31 | |||
| 32 | const url = getDeleteActivityPubUrl(byActor.url) | ||
| 33 | const activity = buildDeleteActivity(url, byActor.url, byActor) | ||
| 34 | |||
| 35 | const actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) | ||
| 36 | |||
| 37 | // In case the actor did not have any videos | ||
| 38 | const serverActor = await getServerActor() | ||
| 39 | actorsInvolved.push(serverActor) | ||
| 40 | |||
| 41 | actorsInvolved.push(byActor) | ||
| 42 | |||
| 43 | return broadcastToFollowers({ | ||
| 44 | data: activity, | ||
| 45 | byActor, | ||
| 46 | toFollowersOf: actorsInvolved, | ||
| 47 | contextType: 'Delete', | ||
| 48 | transaction | ||
| 49 | }) | ||
| 50 | } | ||
| 51 | |||
| 52 | async function sendDeleteVideoComment (videoComment: MCommentOwnerVideo, transaction: Transaction) { | ||
| 53 | logger.info('Creating job to send delete of comment %s.', videoComment.url) | ||
| 54 | |||
| 55 | const isVideoOrigin = videoComment.Video.isOwned() | ||
| 56 | |||
| 57 | const url = getDeleteActivityPubUrl(videoComment.url) | ||
| 58 | const byActor = videoComment.isOwned() | ||
| 59 | ? videoComment.Account.Actor | ||
| 60 | : videoComment.Video.VideoChannel.Account.Actor | ||
| 61 | |||
| 62 | const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, transaction) | ||
| 63 | const threadParentCommentsFiltered = threadParentComments.filter(c => !c.isDeleted()) | ||
| 64 | |||
| 65 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, transaction) | ||
| 66 | actorsInvolvedInComment.push(byActor) // Add the actor that commented the video | ||
| 67 | |||
| 68 | const audience = getVideoCommentAudience(videoComment, threadParentCommentsFiltered, actorsInvolvedInComment, isVideoOrigin) | ||
| 69 | const activity = buildDeleteActivity(url, videoComment.url, byActor, audience) | ||
| 70 | |||
| 71 | // This was a reply, send it to the parent actors | ||
| 72 | const actorsException = [ byActor ] | ||
| 73 | await broadcastToActors({ | ||
| 74 | data: activity, | ||
| 75 | byActor, | ||
| 76 | toActors: threadParentCommentsFiltered.map(c => c.Account.Actor), | ||
| 77 | transaction, | ||
| 78 | contextType: 'Delete', | ||
| 79 | actorsException | ||
| 80 | }) | ||
| 81 | |||
| 82 | // Broadcast to our followers | ||
| 83 | await broadcastToFollowers({ | ||
| 84 | data: activity, | ||
| 85 | byActor, | ||
| 86 | toFollowersOf: [ byActor ], | ||
| 87 | contextType: 'Delete', | ||
| 88 | transaction | ||
| 89 | }) | ||
| 90 | |||
| 91 | // Send to actors involved in the comment | ||
| 92 | if (isVideoOrigin) { | ||
| 93 | return broadcastToFollowers({ | ||
| 94 | data: activity, | ||
| 95 | byActor, | ||
| 96 | toFollowersOf: actorsInvolvedInComment, | ||
| 97 | transaction, | ||
| 98 | contextType: 'Delete', | ||
| 99 | actorsException | ||
| 100 | }) | ||
| 101 | } | ||
| 102 | |||
| 103 | // Send to origin | ||
| 104 | return transaction.afterCommit(() => { | ||
| 105 | return unicastTo({ | ||
| 106 | data: activity, | ||
| 107 | byActor, | ||
| 108 | toActorUrl: videoComment.Video.VideoChannel.Account.Actor.getSharedInbox(), | ||
| 109 | contextType: 'Delete' | ||
| 110 | }) | ||
| 111 | }) | ||
| 112 | } | ||
| 113 | |||
| 114 | async function sendDeleteVideoPlaylist (videoPlaylist: MVideoPlaylistFullSummary, transaction: Transaction) { | ||
| 115 | logger.info('Creating job to send delete of playlist %s.', videoPlaylist.url) | ||
| 116 | |||
| 117 | const byActor = videoPlaylist.OwnerAccount.Actor | ||
| 118 | |||
| 119 | const url = getDeleteActivityPubUrl(videoPlaylist.url) | ||
| 120 | const activity = buildDeleteActivity(url, videoPlaylist.url, byActor) | ||
| 121 | |||
| 122 | const serverActor = await getServerActor() | ||
| 123 | const toFollowersOf = [ byActor, serverActor ] | ||
| 124 | |||
| 125 | if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) | ||
| 126 | |||
| 127 | return broadcastToFollowers({ | ||
| 128 | data: activity, | ||
| 129 | byActor, | ||
| 130 | toFollowersOf, | ||
| 131 | contextType: 'Delete', | ||
| 132 | transaction | ||
| 133 | }) | ||
| 134 | } | ||
| 135 | |||
| 136 | // --------------------------------------------------------------------------- | ||
| 137 | |||
| 138 | export { | ||
| 139 | sendDeleteVideo, | ||
| 140 | sendDeleteActor, | ||
| 141 | sendDeleteVideoComment, | ||
| 142 | sendDeleteVideoPlaylist | ||
| 143 | } | ||
| 144 | |||
| 145 | // --------------------------------------------------------------------------- | ||
| 146 | |||
| 147 | function buildDeleteActivity (url: string, object: string, byActor: MActorUrl, audience?: ActivityAudience): ActivityDelete { | ||
| 148 | const activity = { | ||
| 149 | type: 'Delete' as 'Delete', | ||
| 150 | id: url, | ||
| 151 | actor: byActor.url, | ||
| 152 | object | ||
| 153 | } | ||
| 154 | |||
| 155 | if (audience) return audiencify(activity, audience) | ||
| 156 | |||
| 157 | return activity | ||
| 158 | } | ||
diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts deleted file mode 100644 index 959e74823..000000000 --- a/server/lib/activitypub/send/send-dislike.ts +++ /dev/null | |||
| @@ -1,40 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ActivityAudience, ActivityDislike } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models' | ||
| 5 | import { audiencify, getAudience } from '../audience' | ||
| 6 | import { getVideoDislikeActivityPubUrlByLocalActor } from '../url' | ||
| 7 | import { sendVideoActivityToOrigin } from './shared/send-utils' | ||
| 8 | |||
| 9 | function sendDislike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { | ||
| 10 | logger.info('Creating job to dislike %s.', video.url) | ||
| 11 | |||
| 12 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 13 | const url = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) | ||
| 14 | |||
| 15 | return buildDislikeActivity(url, byActor, video, audience) | ||
| 16 | } | ||
| 17 | |||
| 18 | return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) | ||
| 19 | } | ||
| 20 | |||
| 21 | function buildDislikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityDislike { | ||
| 22 | if (!audience) audience = getAudience(byActor) | ||
| 23 | |||
| 24 | return audiencify( | ||
| 25 | { | ||
| 26 | id: url, | ||
| 27 | type: 'Dislike' as 'Dislike', | ||
| 28 | actor: byActor.url, | ||
| 29 | object: video.url | ||
| 30 | }, | ||
| 31 | audience | ||
| 32 | ) | ||
| 33 | } | ||
| 34 | |||
| 35 | // --------------------------------------------------------------------------- | ||
| 36 | |||
| 37 | export { | ||
| 38 | sendDislike, | ||
| 39 | buildDislikeActivity | ||
| 40 | } | ||
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts deleted file mode 100644 index 138eb5adc..000000000 --- a/server/lib/activitypub/send/send-flag.ts +++ /dev/null | |||
| @@ -1,42 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ActivityAudience, ActivityFlag } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { MAbuseAP, MAccountLight, MActor } from '../../../types/models' | ||
| 5 | import { audiencify, getAudience } from '../audience' | ||
| 6 | import { getLocalAbuseActivityPubUrl } from '../url' | ||
| 7 | import { unicastTo } from './shared/send-utils' | ||
| 8 | |||
| 9 | function sendAbuse (byActor: MActor, abuse: MAbuseAP, flaggedAccount: MAccountLight, t: Transaction) { | ||
| 10 | if (!flaggedAccount.Actor.serverId) return // Local user | ||
| 11 | |||
| 12 | const url = getLocalAbuseActivityPubUrl(abuse) | ||
| 13 | |||
| 14 | logger.info('Creating job to send abuse %s.', url) | ||
| 15 | |||
| 16 | // Custom audience, we only send the abuse to the origin instance | ||
| 17 | const audience = { to: [ flaggedAccount.Actor.url ], cc: [] } | ||
| 18 | const flagActivity = buildFlagActivity(url, byActor, abuse, audience) | ||
| 19 | |||
| 20 | return t.afterCommit(() => { | ||
| 21 | return unicastTo({ | ||
| 22 | data: flagActivity, | ||
| 23 | byActor, | ||
| 24 | toActorUrl: flaggedAccount.Actor.getSharedInbox(), | ||
| 25 | contextType: 'Flag' | ||
| 26 | }) | ||
| 27 | }) | ||
| 28 | } | ||
| 29 | |||
| 30 | function buildFlagActivity (url: string, byActor: MActor, abuse: MAbuseAP, audience: ActivityAudience): ActivityFlag { | ||
| 31 | if (!audience) audience = getAudience(byActor) | ||
| 32 | |||
| 33 | const activity = { id: url, actor: byActor.url, ...abuse.toActivityPubObject() } | ||
| 34 | |||
| 35 | return audiencify(activity, audience) | ||
| 36 | } | ||
| 37 | |||
| 38 | // --------------------------------------------------------------------------- | ||
| 39 | |||
| 40 | export { | ||
| 41 | sendAbuse | ||
| 42 | } | ||
diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts deleted file mode 100644 index 57501dadb..000000000 --- a/server/lib/activitypub/send/send-follow.ts +++ /dev/null | |||
| @@ -1,37 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ActivityFollow } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { MActor, MActorFollowActors } from '../../../types/models' | ||
| 5 | import { unicastTo } from './shared/send-utils' | ||
| 6 | |||
| 7 | function sendFollow (actorFollow: MActorFollowActors, t: Transaction) { | ||
| 8 | const me = actorFollow.ActorFollower | ||
| 9 | const following = actorFollow.ActorFollowing | ||
| 10 | |||
| 11 | // Same server as ours | ||
| 12 | if (!following.serverId) return | ||
| 13 | |||
| 14 | logger.info('Creating job to send follow request to %s.', following.url) | ||
| 15 | |||
| 16 | const data = buildFollowActivity(actorFollow.url, me, following) | ||
| 17 | |||
| 18 | return t.afterCommit(() => { | ||
| 19 | return unicastTo({ data, byActor: me, toActorUrl: following.inboxUrl, contextType: 'Follow' }) | ||
| 20 | }) | ||
| 21 | } | ||
| 22 | |||
| 23 | function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow { | ||
| 24 | return { | ||
| 25 | type: 'Follow', | ||
| 26 | id: url, | ||
| 27 | actor: byActor.url, | ||
| 28 | object: targetActor.url | ||
| 29 | } | ||
| 30 | } | ||
| 31 | |||
| 32 | // --------------------------------------------------------------------------- | ||
| 33 | |||
| 34 | export { | ||
| 35 | sendFollow, | ||
| 36 | buildFollowActivity | ||
| 37 | } | ||
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts deleted file mode 100644 index 46c9fdec9..000000000 --- a/server/lib/activitypub/send/send-like.ts +++ /dev/null | |||
| @@ -1,40 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ActivityAudience, ActivityLike } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models' | ||
| 5 | import { audiencify, getAudience } from '../audience' | ||
| 6 | import { getVideoLikeActivityPubUrlByLocalActor } from '../url' | ||
| 7 | import { sendVideoActivityToOrigin } from './shared/send-utils' | ||
| 8 | |||
| 9 | function sendLike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { | ||
| 10 | logger.info('Creating job to like %s.', video.url) | ||
| 11 | |||
| 12 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 13 | const url = getVideoLikeActivityPubUrlByLocalActor(byActor, video) | ||
| 14 | |||
| 15 | return buildLikeActivity(url, byActor, video, audience) | ||
| 16 | } | ||
| 17 | |||
| 18 | return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) | ||
| 19 | } | ||
| 20 | |||
| 21 | function buildLikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityLike { | ||
| 22 | if (!audience) audience = getAudience(byActor) | ||
| 23 | |||
| 24 | return audiencify( | ||
| 25 | { | ||
| 26 | id: url, | ||
| 27 | type: 'Like' as 'Like', | ||
| 28 | actor: byActor.url, | ||
| 29 | object: video.url | ||
| 30 | }, | ||
| 31 | audience | ||
| 32 | ) | ||
| 33 | } | ||
| 34 | |||
| 35 | // --------------------------------------------------------------------------- | ||
| 36 | |||
| 37 | export { | ||
| 38 | sendLike, | ||
| 39 | buildLikeActivity | ||
| 40 | } | ||
diff --git a/server/lib/activitypub/send/send-reject.ts b/server/lib/activitypub/send/send-reject.ts deleted file mode 100644 index a5f8c2ecf..000000000 --- a/server/lib/activitypub/send/send-reject.ts +++ /dev/null | |||
| @@ -1,39 +0,0 @@ | |||
| 1 | import { ActivityFollow, ActivityReject } from '@shared/models' | ||
| 2 | import { logger } from '../../../helpers/logger' | ||
| 3 | import { MActor } from '../../../types/models' | ||
| 4 | import { getLocalActorFollowRejectActivityPubUrl } from '../url' | ||
| 5 | import { buildFollowActivity } from './send-follow' | ||
| 6 | import { unicastTo } from './shared/send-utils' | ||
| 7 | |||
| 8 | function sendReject (followUrl: string, follower: MActor, following: MActor) { | ||
| 9 | if (!follower.serverId) { // This should never happen | ||
| 10 | logger.warn('Do not sending reject to local follower.') | ||
| 11 | return | ||
| 12 | } | ||
| 13 | |||
| 14 | logger.info('Creating job to reject follower %s.', follower.url) | ||
| 15 | |||
| 16 | const followData = buildFollowActivity(followUrl, follower, following) | ||
| 17 | |||
| 18 | const url = getLocalActorFollowRejectActivityPubUrl() | ||
| 19 | const data = buildRejectActivity(url, following, followData) | ||
| 20 | |||
| 21 | return unicastTo({ data, byActor: following, toActorUrl: follower.inboxUrl, contextType: 'Reject' }) | ||
| 22 | } | ||
| 23 | |||
| 24 | // --------------------------------------------------------------------------- | ||
| 25 | |||
| 26 | export { | ||
| 27 | sendReject | ||
| 28 | } | ||
| 29 | |||
| 30 | // --------------------------------------------------------------------------- | ||
| 31 | |||
| 32 | function buildRejectActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityReject { | ||
| 33 | return { | ||
| 34 | type: 'Reject', | ||
| 35 | id: url, | ||
| 36 | actor: byActor.url, | ||
| 37 | object: followActivityData | ||
| 38 | } | ||
| 39 | } | ||
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts deleted file mode 100644 index b0b48c9c4..000000000 --- a/server/lib/activitypub/send/send-undo.ts +++ /dev/null | |||
| @@ -1,172 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ActivityAudience, ActivityDislike, ActivityLike, ActivityUndo, ActivityUndoObject, ContextType } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { VideoModel } from '../../../models/video/video' | ||
| 5 | import { | ||
| 6 | MActor, | ||
| 7 | MActorAudience, | ||
| 8 | MActorFollowActors, | ||
| 9 | MActorLight, | ||
| 10 | MVideo, | ||
| 11 | MVideoAccountLight, | ||
| 12 | MVideoRedundancyVideo, | ||
| 13 | MVideoShare | ||
| 14 | } from '../../../types/models' | ||
| 15 | import { audiencify, getAudience } from '../audience' | ||
| 16 | import { getUndoActivityPubUrl, getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from '../url' | ||
| 17 | import { buildAnnounceWithVideoAudience } from './send-announce' | ||
| 18 | import { buildCreateActivity } from './send-create' | ||
| 19 | import { buildDislikeActivity } from './send-dislike' | ||
| 20 | import { buildFollowActivity } from './send-follow' | ||
| 21 | import { buildLikeActivity } from './send-like' | ||
| 22 | import { broadcastToFollowers, sendVideoActivityToOrigin, sendVideoRelatedActivity, unicastTo } from './shared/send-utils' | ||
| 23 | |||
| 24 | function sendUndoFollow (actorFollow: MActorFollowActors, t: Transaction) { | ||
| 25 | const me = actorFollow.ActorFollower | ||
| 26 | const following = actorFollow.ActorFollowing | ||
| 27 | |||
| 28 | // Same server as ours | ||
| 29 | if (!following.serverId) return | ||
| 30 | |||
| 31 | logger.info('Creating job to send an unfollow request to %s.', following.url) | ||
| 32 | |||
| 33 | const undoUrl = getUndoActivityPubUrl(actorFollow.url) | ||
| 34 | |||
| 35 | const followActivity = buildFollowActivity(actorFollow.url, me, following) | ||
| 36 | const undoActivity = undoActivityData(undoUrl, me, followActivity) | ||
| 37 | |||
| 38 | t.afterCommit(() => { | ||
| 39 | return unicastTo({ | ||
| 40 | data: undoActivity, | ||
| 41 | byActor: me, | ||
| 42 | toActorUrl: following.inboxUrl, | ||
| 43 | contextType: 'Follow' | ||
| 44 | }) | ||
| 45 | }) | ||
| 46 | } | ||
| 47 | |||
| 48 | // --------------------------------------------------------------------------- | ||
| 49 | |||
| 50 | async function sendUndoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { | ||
| 51 | logger.info('Creating job to undo announce %s.', videoShare.url) | ||
| 52 | |||
| 53 | const undoUrl = getUndoActivityPubUrl(videoShare.url) | ||
| 54 | |||
| 55 | const { activity: announce, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) | ||
| 56 | const undoActivity = undoActivityData(undoUrl, byActor, announce) | ||
| 57 | |||
| 58 | return broadcastToFollowers({ | ||
| 59 | data: undoActivity, | ||
| 60 | byActor, | ||
| 61 | toFollowersOf: actorsInvolvedInVideo, | ||
| 62 | transaction, | ||
| 63 | actorsException: [ byActor ], | ||
| 64 | contextType: 'Announce' | ||
| 65 | }) | ||
| 66 | } | ||
| 67 | |||
| 68 | async function sendUndoCacheFile (byActor: MActor, redundancyModel: MVideoRedundancyVideo, transaction: Transaction) { | ||
| 69 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | ||
| 70 | |||
| 71 | const associatedVideo = redundancyModel.getVideo() | ||
| 72 | if (!associatedVideo) { | ||
| 73 | logger.warn('Cannot send undo activity for redundancy %s: no video files associated.', redundancyModel.url) | ||
| 74 | return | ||
| 75 | } | ||
| 76 | |||
| 77 | const video = await VideoModel.loadFull(associatedVideo.id) | ||
| 78 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | ||
| 79 | |||
| 80 | return sendUndoVideoRelatedActivity({ | ||
| 81 | byActor, | ||
| 82 | video, | ||
| 83 | url: redundancyModel.url, | ||
| 84 | activity: createActivity, | ||
| 85 | contextType: 'CacheFile', | ||
| 86 | transaction | ||
| 87 | }) | ||
| 88 | } | ||
| 89 | |||
| 90 | // --------------------------------------------------------------------------- | ||
| 91 | |||
| 92 | async function sendUndoLike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { | ||
| 93 | logger.info('Creating job to undo a like of video %s.', video.url) | ||
| 94 | |||
| 95 | const likeUrl = getVideoLikeActivityPubUrlByLocalActor(byActor, video) | ||
| 96 | const likeActivity = buildLikeActivity(likeUrl, byActor, video) | ||
| 97 | |||
| 98 | return sendUndoVideoRateToOriginActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t }) | ||
| 99 | } | ||
| 100 | |||
| 101 | async function sendUndoDislike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { | ||
| 102 | logger.info('Creating job to undo a dislike of video %s.', video.url) | ||
| 103 | |||
| 104 | const dislikeUrl = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) | ||
| 105 | const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) | ||
| 106 | |||
| 107 | return sendUndoVideoRateToOriginActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) | ||
| 108 | } | ||
| 109 | |||
| 110 | // --------------------------------------------------------------------------- | ||
| 111 | |||
| 112 | export { | ||
| 113 | sendUndoFollow, | ||
| 114 | sendUndoLike, | ||
| 115 | sendUndoDislike, | ||
| 116 | sendUndoAnnounce, | ||
| 117 | sendUndoCacheFile | ||
| 118 | } | ||
| 119 | |||
| 120 | // --------------------------------------------------------------------------- | ||
| 121 | |||
| 122 | function undoActivityData <T extends ActivityUndoObject> ( | ||
| 123 | url: string, | ||
| 124 | byActor: MActorAudience, | ||
| 125 | object: T, | ||
| 126 | audience?: ActivityAudience | ||
| 127 | ): ActivityUndo<T> { | ||
| 128 | if (!audience) audience = getAudience(byActor) | ||
| 129 | |||
| 130 | return audiencify( | ||
| 131 | { | ||
| 132 | type: 'Undo' as 'Undo', | ||
| 133 | id: url, | ||
| 134 | actor: byActor.url, | ||
| 135 | object | ||
| 136 | }, | ||
| 137 | audience | ||
| 138 | ) | ||
| 139 | } | ||
| 140 | |||
| 141 | async function sendUndoVideoRelatedActivity (options: { | ||
| 142 | byActor: MActor | ||
| 143 | video: MVideoAccountLight | ||
| 144 | url: string | ||
| 145 | activity: ActivityUndoObject | ||
| 146 | contextType: ContextType | ||
| 147 | transaction: Transaction | ||
| 148 | }) { | ||
| 149 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 150 | const undoUrl = getUndoActivityPubUrl(options.url) | ||
| 151 | |||
| 152 | return undoActivityData(undoUrl, options.byActor, options.activity, audience) | ||
| 153 | } | ||
| 154 | |||
| 155 | return sendVideoRelatedActivity(activityBuilder, options) | ||
| 156 | } | ||
| 157 | |||
| 158 | async function sendUndoVideoRateToOriginActivity (options: { | ||
| 159 | byActor: MActor | ||
| 160 | video: MVideoAccountLight | ||
| 161 | url: string | ||
| 162 | activity: ActivityLike | ActivityDislike | ||
| 163 | transaction: Transaction | ||
| 164 | }) { | ||
| 165 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 166 | const undoUrl = getUndoActivityPubUrl(options.url) | ||
| 167 | |||
| 168 | return undoActivityData(undoUrl, options.byActor, options.activity, audience) | ||
| 169 | } | ||
| 170 | |||
| 171 | return sendVideoActivityToOrigin(activityBuilder, { ...options, contextType: 'Rate' }) | ||
| 172 | } | ||
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts deleted file mode 100644 index f3fb741c6..000000000 --- a/server/lib/activitypub/send/send-update.ts +++ /dev/null | |||
| @@ -1,157 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { getServerActor } from '@server/models/application/application' | ||
| 3 | import { ActivityAudience, ActivityUpdate, ActivityUpdateObject, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' | ||
| 4 | import { logger } from '../../../helpers/logger' | ||
| 5 | import { AccountModel } from '../../../models/account/account' | ||
| 6 | import { VideoModel } from '../../../models/video/video' | ||
| 7 | import { VideoShareModel } from '../../../models/video/video-share' | ||
| 8 | import { | ||
| 9 | MAccountDefault, | ||
| 10 | MActor, | ||
| 11 | MActorLight, | ||
| 12 | MChannelDefault, | ||
| 13 | MVideoAPLight, | ||
| 14 | MVideoPlaylistFull, | ||
| 15 | MVideoRedundancyVideo | ||
| 16 | } from '../../../types/models' | ||
| 17 | import { audiencify, getAudience } from '../audience' | ||
| 18 | import { getUpdateActivityPubUrl } from '../url' | ||
| 19 | import { getActorsInvolvedInVideo } from './shared' | ||
| 20 | import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils' | ||
| 21 | |||
| 22 | async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) { | ||
| 23 | if (!videoArg.hasPrivacyForFederation()) return undefined | ||
| 24 | |||
| 25 | const video = await videoArg.lightAPToFullAP(transaction) | ||
| 26 | |||
| 27 | logger.info('Creating job to update video %s.', video.url) | ||
| 28 | |||
| 29 | const byActor = overriddenByActor || video.VideoChannel.Account.Actor | ||
| 30 | |||
| 31 | const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) | ||
| 32 | |||
| 33 | const videoObject = await video.toActivityPubObject() | ||
| 34 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) | ||
| 35 | |||
| 36 | const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience) | ||
| 37 | |||
| 38 | const actorsInvolved = await getActorsInvolvedInVideo(video, transaction) | ||
| 39 | if (overriddenByActor) actorsInvolved.push(overriddenByActor) | ||
| 40 | |||
| 41 | return broadcastToFollowers({ | ||
| 42 | data: updateActivity, | ||
| 43 | byActor, | ||
| 44 | toFollowersOf: actorsInvolved, | ||
| 45 | contextType: 'Video', | ||
| 46 | transaction | ||
| 47 | }) | ||
| 48 | } | ||
| 49 | |||
| 50 | async function sendUpdateActor (accountOrChannel: MChannelDefault | MAccountDefault, transaction: Transaction) { | ||
| 51 | const byActor = accountOrChannel.Actor | ||
| 52 | |||
| 53 | logger.info('Creating job to update actor %s.', byActor.url) | ||
| 54 | |||
| 55 | const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) | ||
| 56 | const accountOrChannelObject = await (accountOrChannel as any).toActivityPubObject() // FIXME: typescript bug? | ||
| 57 | const audience = getAudience(byActor) | ||
| 58 | const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience) | ||
| 59 | |||
| 60 | let actorsInvolved: MActor[] | ||
| 61 | if (accountOrChannel instanceof AccountModel) { | ||
| 62 | // Actors that shared my videos are involved too | ||
| 63 | actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) | ||
| 64 | } else { | ||
| 65 | // Actors that shared videos of my channel are involved too | ||
| 66 | actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, transaction) | ||
| 67 | } | ||
| 68 | |||
| 69 | actorsInvolved.push(byActor) | ||
| 70 | |||
| 71 | return broadcastToFollowers({ | ||
| 72 | data: updateActivity, | ||
| 73 | byActor, | ||
| 74 | toFollowersOf: actorsInvolved, | ||
| 75 | transaction, | ||
| 76 | contextType: 'Actor' | ||
| 77 | }) | ||
| 78 | } | ||
| 79 | |||
| 80 | async function sendUpdateCacheFile (byActor: MActorLight, redundancyModel: MVideoRedundancyVideo) { | ||
| 81 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | ||
| 82 | |||
| 83 | const associatedVideo = redundancyModel.getVideo() | ||
| 84 | if (!associatedVideo) { | ||
| 85 | logger.warn('Cannot send update activity for redundancy %s: no video files associated.', redundancyModel.url) | ||
| 86 | return | ||
| 87 | } | ||
| 88 | |||
| 89 | const video = await VideoModel.loadFull(associatedVideo.id) | ||
| 90 | |||
| 91 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 92 | const redundancyObject = redundancyModel.toActivityPubObject() | ||
| 93 | const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) | ||
| 94 | |||
| 95 | return buildUpdateActivity(url, byActor, redundancyObject, audience) | ||
| 96 | } | ||
| 97 | |||
| 98 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'CacheFile' }) | ||
| 99 | } | ||
| 100 | |||
| 101 | async function sendUpdateVideoPlaylist (videoPlaylist: MVideoPlaylistFull, transaction: Transaction) { | ||
| 102 | if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined | ||
| 103 | |||
| 104 | const byActor = videoPlaylist.OwnerAccount.Actor | ||
| 105 | |||
| 106 | logger.info('Creating job to update video playlist %s.', videoPlaylist.url) | ||
| 107 | |||
| 108 | const url = getUpdateActivityPubUrl(videoPlaylist.url, videoPlaylist.updatedAt.toISOString()) | ||
| 109 | |||
| 110 | const object = await videoPlaylist.toActivityPubObject(null, transaction) | ||
| 111 | const audience = getAudience(byActor, videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC) | ||
| 112 | |||
| 113 | const updateActivity = buildUpdateActivity(url, byActor, object, audience) | ||
| 114 | |||
| 115 | const serverActor = await getServerActor() | ||
| 116 | const toFollowersOf = [ byActor, serverActor ] | ||
| 117 | |||
| 118 | if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) | ||
| 119 | |||
| 120 | return broadcastToFollowers({ | ||
| 121 | data: updateActivity, | ||
| 122 | byActor, | ||
| 123 | toFollowersOf, | ||
| 124 | transaction, | ||
| 125 | contextType: 'Playlist' | ||
| 126 | }) | ||
| 127 | } | ||
| 128 | |||
| 129 | // --------------------------------------------------------------------------- | ||
| 130 | |||
| 131 | export { | ||
| 132 | sendUpdateActor, | ||
| 133 | sendUpdateVideo, | ||
| 134 | sendUpdateCacheFile, | ||
| 135 | sendUpdateVideoPlaylist | ||
| 136 | } | ||
| 137 | |||
| 138 | // --------------------------------------------------------------------------- | ||
| 139 | |||
| 140 | function buildUpdateActivity ( | ||
| 141 | url: string, | ||
| 142 | byActor: MActorLight, | ||
| 143 | object: ActivityUpdateObject, | ||
| 144 | audience?: ActivityAudience | ||
| 145 | ): ActivityUpdate<ActivityUpdateObject> { | ||
| 146 | if (!audience) audience = getAudience(byActor) | ||
| 147 | |||
| 148 | return audiencify( | ||
| 149 | { | ||
| 150 | type: 'Update' as 'Update', | ||
| 151 | id: url, | ||
| 152 | actor: byActor.url, | ||
| 153 | object: audiencify(object, audience) | ||
| 154 | }, | ||
| 155 | audience | ||
| 156 | ) | ||
| 157 | } | ||
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts deleted file mode 100644 index bf3451603..000000000 --- a/server/lib/activitypub/send/send-view.ts +++ /dev/null | |||
| @@ -1,62 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' | ||
| 3 | import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models' | ||
| 4 | import { ActivityAudience, ActivityView } from '@shared/models' | ||
| 5 | import { logger } from '../../../helpers/logger' | ||
| 6 | import { audiencify, getAudience } from '../audience' | ||
| 7 | import { getLocalVideoViewActivityPubUrl } from '../url' | ||
| 8 | import { sendVideoRelatedActivity } from './shared/send-utils' | ||
| 9 | |||
| 10 | type ViewType = 'view' | 'viewer' | ||
| 11 | |||
| 12 | async function sendView (options: { | ||
| 13 | byActor: MActorLight | ||
| 14 | type: ViewType | ||
| 15 | video: MVideoImmutable | ||
| 16 | viewerIdentifier: string | ||
| 17 | transaction?: Transaction | ||
| 18 | }) { | ||
| 19 | const { byActor, type, video, viewerIdentifier, transaction } = options | ||
| 20 | |||
| 21 | logger.info('Creating job to send %s of %s.', type, video.url) | ||
| 22 | |||
| 23 | const activityBuilder = (audience: ActivityAudience) => { | ||
| 24 | const url = getLocalVideoViewActivityPubUrl(byActor, video, viewerIdentifier) | ||
| 25 | |||
| 26 | return buildViewActivity({ url, byActor, video, audience, type }) | ||
| 27 | } | ||
| 28 | |||
| 29 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View', parallelizable: true }) | ||
| 30 | } | ||
| 31 | |||
| 32 | // --------------------------------------------------------------------------- | ||
| 33 | |||
| 34 | export { | ||
| 35 | sendView | ||
| 36 | } | ||
| 37 | |||
| 38 | // --------------------------------------------------------------------------- | ||
| 39 | |||
| 40 | function buildViewActivity (options: { | ||
| 41 | url: string | ||
| 42 | byActor: MActorAudience | ||
| 43 | video: MVideoUrl | ||
| 44 | type: ViewType | ||
| 45 | audience?: ActivityAudience | ||
| 46 | }): ActivityView { | ||
| 47 | const { url, byActor, type, video, audience = getAudience(byActor) } = options | ||
| 48 | |||
| 49 | return audiencify( | ||
| 50 | { | ||
| 51 | id: url, | ||
| 52 | type: 'View' as 'View', | ||
| 53 | actor: byActor.url, | ||
| 54 | object: video.url, | ||
| 55 | |||
| 56 | expires: type === 'viewer' | ||
| 57 | ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString() | ||
| 58 | : undefined | ||
| 59 | }, | ||
| 60 | audience | ||
| 61 | ) | ||
| 62 | } | ||
diff --git a/server/lib/activitypub/send/shared/audience-utils.ts b/server/lib/activitypub/send/shared/audience-utils.ts deleted file mode 100644 index 2f6b0741d..000000000 --- a/server/lib/activitypub/send/shared/audience-utils.ts +++ /dev/null | |||
| @@ -1,74 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ACTIVITY_PUB } from '@server/initializers/constants' | ||
| 3 | import { ActorModel } from '@server/models/actor/actor' | ||
| 4 | import { VideoModel } from '@server/models/video/video' | ||
| 5 | import { VideoShareModel } from '@server/models/video/video-share' | ||
| 6 | import { MActorFollowersUrl, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' | ||
| 7 | import { ActivityAudience } from '@shared/models' | ||
| 8 | |||
| 9 | function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { | ||
| 10 | return { | ||
| 11 | to: [ accountActor.url ], | ||
| 12 | cc: actorsInvolvedInVideo.map(a => a.followersUrl) | ||
| 13 | } | ||
| 14 | } | ||
| 15 | |||
| 16 | function getVideoCommentAudience ( | ||
| 17 | videoComment: MCommentOwnerVideo, | ||
| 18 | threadParentComments: MCommentOwner[], | ||
| 19 | actorsInvolvedInVideo: MActorFollowersUrl[], | ||
| 20 | isOrigin = false | ||
| 21 | ): ActivityAudience { | ||
| 22 | const to = [ ACTIVITY_PUB.PUBLIC ] | ||
| 23 | const cc: string[] = [] | ||
| 24 | |||
| 25 | // Owner of the video we comment | ||
| 26 | if (isOrigin === false) { | ||
| 27 | cc.push(videoComment.Video.VideoChannel.Account.Actor.url) | ||
| 28 | } | ||
| 29 | |||
| 30 | // Followers of the poster | ||
| 31 | cc.push(videoComment.Account.Actor.followersUrl) | ||
| 32 | |||
| 33 | // Send to actors we reply to | ||
| 34 | for (const parentComment of threadParentComments) { | ||
| 35 | if (parentComment.isDeleted()) continue | ||
| 36 | |||
| 37 | cc.push(parentComment.Account.Actor.url) | ||
| 38 | } | ||
| 39 | |||
| 40 | return { | ||
| 41 | to, | ||
| 42 | cc: cc.concat(actorsInvolvedInVideo.map(a => a.followersUrl)) | ||
| 43 | } | ||
| 44 | } | ||
| 45 | |||
| 46 | function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[]): ActivityAudience { | ||
| 47 | return { | ||
| 48 | to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), | ||
| 49 | cc: [] | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { | ||
| 54 | const actors = await VideoShareModel.listActorIdsAndFollowerUrlsByShare(video.id, t) | ||
| 55 | |||
| 56 | const videoAll = video as VideoModel | ||
| 57 | |||
| 58 | const videoActor = videoAll.VideoChannel?.Account | ||
| 59 | ? videoAll.VideoChannel.Account.Actor | ||
| 60 | : await ActorModel.loadAccountActorFollowerUrlByVideoId(video.id, t) | ||
| 61 | |||
| 62 | actors.push(videoActor) | ||
| 63 | |||
| 64 | return actors | ||
| 65 | } | ||
| 66 | |||
| 67 | // --------------------------------------------------------------------------- | ||
| 68 | |||
| 69 | export { | ||
| 70 | getOriginVideoAudience, | ||
| 71 | getActorsInvolvedInVideo, | ||
| 72 | getAudienceFromFollowersOf, | ||
| 73 | getVideoCommentAudience | ||
| 74 | } | ||
diff --git a/server/lib/activitypub/send/shared/index.ts b/server/lib/activitypub/send/shared/index.ts deleted file mode 100644 index bda579115..000000000 --- a/server/lib/activitypub/send/shared/index.ts +++ /dev/null | |||
| @@ -1,2 +0,0 @@ | |||
| 1 | export * from './audience-utils' | ||
| 2 | export * from './send-utils' | ||
diff --git a/server/lib/activitypub/send/shared/send-utils.ts b/server/lib/activitypub/send/shared/send-utils.ts deleted file mode 100644 index 2bc1ef8f5..000000000 --- a/server/lib/activitypub/send/shared/send-utils.ts +++ /dev/null | |||
| @@ -1,291 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache' | ||
| 3 | import { getServerActor } from '@server/models/application/application' | ||
| 4 | import { Activity, ActivityAudience, ActivitypubHttpBroadcastPayload } from '@shared/models' | ||
| 5 | import { ContextType } from '@shared/models/activitypub/context' | ||
| 6 | import { afterCommitIfTransaction } from '../../../../helpers/database-utils' | ||
| 7 | import { logger } from '../../../../helpers/logger' | ||
| 8 | import { ActorModel } from '../../../../models/actor/actor' | ||
| 9 | import { ActorFollowModel } from '../../../../models/actor/actor-follow' | ||
| 10 | import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../../types/models' | ||
| 11 | import { JobQueue } from '../../../job-queue' | ||
| 12 | import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getOriginVideoAudience } from './audience-utils' | ||
| 13 | |||
| 14 | async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { | ||
| 15 | byActor: MActorLight | ||
| 16 | video: MVideoImmutable | MVideoAccountLight | ||
| 17 | contextType: ContextType | ||
| 18 | parallelizable?: boolean | ||
| 19 | transaction?: Transaction | ||
| 20 | }) { | ||
| 21 | const { byActor, video, transaction, contextType, parallelizable } = options | ||
| 22 | |||
| 23 | // Send to origin | ||
| 24 | if (video.isOwned() === false) { | ||
| 25 | return sendVideoActivityToOrigin(activityBuilder, options) | ||
| 26 | } | ||
| 27 | |||
| 28 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) | ||
| 29 | |||
| 30 | // Send to followers | ||
| 31 | const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) | ||
| 32 | const activity = activityBuilder(audience) | ||
| 33 | |||
| 34 | const actorsException = [ byActor ] | ||
| 35 | |||
| 36 | return broadcastToFollowers({ | ||
| 37 | data: activity, | ||
| 38 | byActor, | ||
| 39 | toFollowersOf: actorsInvolvedInVideo, | ||
| 40 | transaction, | ||
| 41 | actorsException, | ||
| 42 | parallelizable, | ||
| 43 | contextType | ||
| 44 | }) | ||
| 45 | } | ||
| 46 | |||
| 47 | async function sendVideoActivityToOrigin (activityBuilder: (audience: ActivityAudience) => Activity, options: { | ||
| 48 | byActor: MActorLight | ||
| 49 | video: MVideoImmutable | MVideoAccountLight | ||
| 50 | contextType: ContextType | ||
| 51 | |||
| 52 | actorsInvolvedInVideo?: MActorLight[] | ||
| 53 | transaction?: Transaction | ||
| 54 | }) { | ||
| 55 | const { byActor, video, actorsInvolvedInVideo, transaction, contextType } = options | ||
| 56 | |||
| 57 | if (video.isOwned()) throw new Error('Cannot send activity to owned video origin ' + video.url) | ||
| 58 | |||
| 59 | let accountActor: MActorLight = (video as MVideoAccountLight).VideoChannel?.Account?.Actor | ||
| 60 | if (!accountActor) accountActor = await ActorModel.loadAccountActorByVideoId(video.id, transaction) | ||
| 61 | |||
| 62 | const audience = getOriginVideoAudience(accountActor, actorsInvolvedInVideo) | ||
| 63 | const activity = activityBuilder(audience) | ||
| 64 | |||
| 65 | return afterCommitIfTransaction(transaction, () => { | ||
| 66 | return unicastTo({ | ||
| 67 | data: activity, | ||
| 68 | byActor, | ||
| 69 | toActorUrl: accountActor.getSharedInbox(), | ||
| 70 | contextType | ||
| 71 | }) | ||
| 72 | }) | ||
| 73 | } | ||
| 74 | |||
| 75 | // --------------------------------------------------------------------------- | ||
| 76 | |||
| 77 | async function forwardVideoRelatedActivity ( | ||
| 78 | activity: Activity, | ||
| 79 | t: Transaction, | ||
| 80 | followersException: MActorWithInboxes[], | ||
| 81 | video: MVideoId | ||
| 82 | ) { | ||
| 83 | // Mastodon does not add our announces in audience, so we forward to them manually | ||
| 84 | const additionalActors = await getActorsInvolvedInVideo(video, t) | ||
| 85 | const additionalFollowerUrls = additionalActors.map(a => a.followersUrl) | ||
| 86 | |||
| 87 | return forwardActivity(activity, t, followersException, additionalFollowerUrls) | ||
| 88 | } | ||
| 89 | |||
| 90 | async function forwardActivity ( | ||
| 91 | activity: Activity, | ||
| 92 | t: Transaction, | ||
| 93 | followersException: MActorWithInboxes[] = [], | ||
| 94 | additionalFollowerUrls: string[] = [] | ||
| 95 | ) { | ||
| 96 | logger.info('Forwarding activity %s.', activity.id) | ||
| 97 | |||
| 98 | const to = activity.to || [] | ||
| 99 | const cc = activity.cc || [] | ||
| 100 | |||
| 101 | const followersUrls = additionalFollowerUrls | ||
| 102 | for (const dest of to.concat(cc)) { | ||
| 103 | if (dest.endsWith('/followers')) { | ||
| 104 | followersUrls.push(dest) | ||
| 105 | } | ||
| 106 | } | ||
| 107 | |||
| 108 | const toActorFollowers = await ActorModel.listByFollowersUrls(followersUrls, t) | ||
| 109 | const uris = await computeFollowerUris(toActorFollowers, followersException, t) | ||
| 110 | |||
| 111 | if (uris.length === 0) { | ||
| 112 | logger.info('0 followers for %s, no forwarding.', toActorFollowers.map(a => a.id).join(', ')) | ||
| 113 | return undefined | ||
| 114 | } | ||
| 115 | |||
| 116 | logger.debug('Creating forwarding job.', { uris }) | ||
| 117 | |||
| 118 | const payload: ActivitypubHttpBroadcastPayload = { | ||
| 119 | uris, | ||
| 120 | body: activity, | ||
| 121 | contextType: null | ||
| 122 | } | ||
| 123 | return afterCommitIfTransaction(t, () => JobQueue.Instance.createJobAsync({ type: 'activitypub-http-broadcast', payload })) | ||
| 124 | } | ||
| 125 | |||
| 126 | // --------------------------------------------------------------------------- | ||
| 127 | |||
| 128 | async function broadcastToFollowers (options: { | ||
| 129 | data: any | ||
| 130 | byActor: MActorId | ||
| 131 | toFollowersOf: MActorId[] | ||
| 132 | transaction: Transaction | ||
| 133 | contextType: ContextType | ||
| 134 | |||
| 135 | parallelizable?: boolean | ||
| 136 | actorsException?: MActorWithInboxes[] | ||
| 137 | }) { | ||
| 138 | const { data, byActor, toFollowersOf, transaction, contextType, actorsException = [], parallelizable } = options | ||
| 139 | |||
| 140 | const uris = await computeFollowerUris(toFollowersOf, actorsException, transaction) | ||
| 141 | |||
| 142 | return afterCommitIfTransaction(transaction, () => { | ||
| 143 | return broadcastTo({ | ||
| 144 | uris, | ||
| 145 | data, | ||
| 146 | byActor, | ||
| 147 | parallelizable, | ||
| 148 | contextType | ||
| 149 | }) | ||
| 150 | }) | ||
| 151 | } | ||
| 152 | |||
| 153 | async function broadcastToActors (options: { | ||
| 154 | data: any | ||
| 155 | byActor: MActorId | ||
| 156 | toActors: MActor[] | ||
| 157 | transaction: Transaction | ||
| 158 | contextType: ContextType | ||
| 159 | actorsException?: MActorWithInboxes[] | ||
| 160 | }) { | ||
| 161 | const { data, byActor, toActors, transaction, contextType, actorsException = [] } = options | ||
| 162 | |||
| 163 | const uris = await computeUris(toActors, actorsException) | ||
| 164 | |||
| 165 | return afterCommitIfTransaction(transaction, () => { | ||
| 166 | return broadcastTo({ | ||
| 167 | uris, | ||
| 168 | data, | ||
| 169 | byActor, | ||
| 170 | contextType | ||
| 171 | }) | ||
| 172 | }) | ||
| 173 | } | ||
| 174 | |||
| 175 | function broadcastTo (options: { | ||
| 176 | uris: string[] | ||
| 177 | data: any | ||
| 178 | byActor: MActorId | ||
| 179 | contextType: ContextType | ||
| 180 | parallelizable?: boolean // default to false | ||
| 181 | }) { | ||
| 182 | const { uris, data, byActor, contextType, parallelizable } = options | ||
| 183 | |||
| 184 | if (uris.length === 0) return undefined | ||
| 185 | |||
| 186 | const broadcastUris: string[] = [] | ||
| 187 | const unicastUris: string[] = [] | ||
| 188 | |||
| 189 | // Bad URIs could be slow to respond, prefer to process them in a dedicated queue | ||
| 190 | for (const uri of uris) { | ||
| 191 | if (ActorFollowHealthCache.Instance.isBadInbox(uri)) { | ||
| 192 | unicastUris.push(uri) | ||
| 193 | } else { | ||
| 194 | broadcastUris.push(uri) | ||
| 195 | } | ||
| 196 | } | ||
| 197 | |||
| 198 | logger.debug('Creating broadcast job.', { broadcastUris, unicastUris }) | ||
| 199 | |||
| 200 | if (broadcastUris.length !== 0) { | ||
| 201 | const payload = { | ||
| 202 | uris: broadcastUris, | ||
| 203 | signatureActorId: byActor.id, | ||
| 204 | body: data, | ||
| 205 | contextType | ||
| 206 | } | ||
| 207 | |||
| 208 | JobQueue.Instance.createJobAsync({ | ||
| 209 | type: parallelizable | ||
| 210 | ? 'activitypub-http-broadcast-parallel' | ||
| 211 | : 'activitypub-http-broadcast', | ||
| 212 | |||
| 213 | payload | ||
| 214 | }) | ||
| 215 | } | ||
| 216 | |||
| 217 | for (const unicastUri of unicastUris) { | ||
| 218 | const payload = { | ||
| 219 | uri: unicastUri, | ||
| 220 | signatureActorId: byActor.id, | ||
| 221 | body: data, | ||
| 222 | contextType | ||
| 223 | } | ||
| 224 | |||
| 225 | JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) | ||
| 226 | } | ||
| 227 | } | ||
| 228 | |||
| 229 | function unicastTo (options: { | ||
| 230 | data: any | ||
| 231 | byActor: MActorId | ||
| 232 | toActorUrl: string | ||
| 233 | contextType: ContextType | ||
| 234 | }) { | ||
| 235 | const { data, byActor, toActorUrl, contextType } = options | ||
| 236 | |||
| 237 | logger.debug('Creating unicast job.', { uri: toActorUrl }) | ||
| 238 | |||
| 239 | const payload = { | ||
| 240 | uri: toActorUrl, | ||
| 241 | signatureActorId: byActor.id, | ||
| 242 | body: data, | ||
| 243 | contextType | ||
| 244 | } | ||
| 245 | |||
| 246 | JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) | ||
| 247 | } | ||
| 248 | |||
| 249 | // --------------------------------------------------------------------------- | ||
| 250 | |||
| 251 | export { | ||
| 252 | broadcastToFollowers, | ||
| 253 | unicastTo, | ||
| 254 | forwardActivity, | ||
| 255 | broadcastToActors, | ||
| 256 | sendVideoActivityToOrigin, | ||
| 257 | forwardVideoRelatedActivity, | ||
| 258 | sendVideoRelatedActivity | ||
| 259 | } | ||
| 260 | |||
| 261 | // --------------------------------------------------------------------------- | ||
| 262 | |||
| 263 | async function computeFollowerUris (toFollowersOf: MActorId[], actorsException: MActorWithInboxes[], t: Transaction) { | ||
| 264 | const toActorFollowerIds = toFollowersOf.map(a => a.id) | ||
| 265 | |||
| 266 | const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) | ||
| 267 | const sharedInboxesException = await buildSharedInboxesException(actorsException) | ||
| 268 | |||
| 269 | return result.data.filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) | ||
| 270 | } | ||
| 271 | |||
| 272 | async function computeUris (toActors: MActor[], actorsException: MActorWithInboxes[] = []) { | ||
| 273 | const serverActor = await getServerActor() | ||
| 274 | const targetUrls = toActors | ||
| 275 | .filter(a => a.id !== serverActor.id) // Don't send to ourselves | ||
| 276 | .map(a => a.getSharedInbox()) | ||
| 277 | |||
| 278 | const toActorSharedInboxesSet = new Set(targetUrls) | ||
| 279 | |||
| 280 | const sharedInboxesException = await buildSharedInboxesException(actorsException) | ||
| 281 | return Array.from(toActorSharedInboxesSet) | ||
| 282 | .filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) | ||
| 283 | } | ||
| 284 | |||
| 285 | async function buildSharedInboxesException (actorsException: MActorWithInboxes[]) { | ||
| 286 | const serverActor = await getServerActor() | ||
| 287 | |||
| 288 | return actorsException | ||
| 289 | .map(f => f.getSharedInbox()) | ||
| 290 | .concat([ serverActor.sharedInboxUrl ]) | ||
| 291 | } | ||
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts deleted file mode 100644 index 792a73f2a..000000000 --- a/server/lib/activitypub/share.ts +++ /dev/null | |||
| @@ -1,120 +0,0 @@ | |||
| 1 | import { map } from 'bluebird' | ||
| 2 | import { Transaction } from 'sequelize' | ||
| 3 | import { getServerActor } from '@server/models/application/application' | ||
| 4 | import { logger, loggerTagsFactory } from '../../helpers/logger' | ||
| 5 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | ||
| 6 | import { VideoShareModel } from '../../models/video/video-share' | ||
| 7 | import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' | ||
| 8 | import { fetchAP, getAPId } from './activity' | ||
| 9 | import { getOrCreateAPActor } from './actors' | ||
| 10 | import { sendUndoAnnounce, sendVideoAnnounce } from './send' | ||
| 11 | import { checkUrlsSameHost, getLocalVideoAnnounceActivityPubUrl } from './url' | ||
| 12 | |||
| 13 | const lTags = loggerTagsFactory('share') | ||
| 14 | |||
| 15 | async function shareVideoByServerAndChannel (video: MVideoAccountLight, t: Transaction) { | ||
| 16 | if (!video.hasPrivacyForFederation()) return undefined | ||
| 17 | |||
| 18 | return Promise.all([ | ||
| 19 | shareByServer(video, t), | ||
| 20 | shareByVideoChannel(video, t) | ||
| 21 | ]) | ||
| 22 | } | ||
| 23 | |||
| 24 | async function changeVideoChannelShare ( | ||
| 25 | video: MVideoAccountLight, | ||
| 26 | oldVideoChannel: MChannelActorLight, | ||
| 27 | t: Transaction | ||
| 28 | ) { | ||
| 29 | logger.info( | ||
| 30 | 'Updating video channel of video %s: %s -> %s.', video.uuid, oldVideoChannel.name, video.VideoChannel.name, | ||
| 31 | lTags(video.uuid) | ||
| 32 | ) | ||
| 33 | |||
| 34 | await undoShareByVideoChannel(video, oldVideoChannel, t) | ||
| 35 | |||
| 36 | await shareByVideoChannel(video, t) | ||
| 37 | } | ||
| 38 | |||
| 39 | async function addVideoShares (shareUrls: string[], video: MVideoId) { | ||
| 40 | await map(shareUrls, async shareUrl => { | ||
| 41 | try { | ||
| 42 | await addVideoShare(shareUrl, video) | ||
| 43 | } catch (err) { | ||
| 44 | logger.warn('Cannot add share %s.', shareUrl, { err }) | ||
| 45 | } | ||
| 46 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
| 47 | } | ||
| 48 | |||
| 49 | export { | ||
| 50 | changeVideoChannelShare, | ||
| 51 | addVideoShares, | ||
| 52 | shareVideoByServerAndChannel | ||
| 53 | } | ||
| 54 | |||
| 55 | // --------------------------------------------------------------------------- | ||
| 56 | |||
| 57 | async function addVideoShare (shareUrl: string, video: MVideoId) { | ||
| 58 | const { body } = await fetchAP<any>(shareUrl) | ||
| 59 | if (!body?.actor) throw new Error('Body or body actor is invalid') | ||
| 60 | |||
| 61 | const actorUrl = getAPId(body.actor) | ||
| 62 | if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { | ||
| 63 | throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) | ||
| 64 | } | ||
| 65 | |||
| 66 | const actor = await getOrCreateAPActor(actorUrl) | ||
| 67 | |||
| 68 | const entry = { | ||
| 69 | actorId: actor.id, | ||
| 70 | videoId: video.id, | ||
| 71 | url: shareUrl | ||
| 72 | } | ||
| 73 | |||
| 74 | await VideoShareModel.upsert(entry) | ||
| 75 | } | ||
| 76 | |||
| 77 | async function shareByServer (video: MVideo, t: Transaction) { | ||
| 78 | const serverActor = await getServerActor() | ||
| 79 | |||
| 80 | const serverShareUrl = getLocalVideoAnnounceActivityPubUrl(serverActor, video) | ||
| 81 | const [ serverShare ] = await VideoShareModel.findOrCreate({ | ||
| 82 | defaults: { | ||
| 83 | actorId: serverActor.id, | ||
| 84 | videoId: video.id, | ||
| 85 | url: serverShareUrl | ||
| 86 | }, | ||
| 87 | where: { | ||
| 88 | url: serverShareUrl | ||
| 89 | }, | ||
| 90 | transaction: t | ||
| 91 | }) | ||
| 92 | |||
| 93 | return sendVideoAnnounce(serverActor, serverShare, video, t) | ||
| 94 | } | ||
| 95 | |||
| 96 | async function shareByVideoChannel (video: MVideoAccountLight, t: Transaction) { | ||
| 97 | const videoChannelShareUrl = getLocalVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) | ||
| 98 | const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ | ||
| 99 | defaults: { | ||
| 100 | actorId: video.VideoChannel.actorId, | ||
| 101 | videoId: video.id, | ||
| 102 | url: videoChannelShareUrl | ||
| 103 | }, | ||
| 104 | where: { | ||
| 105 | url: videoChannelShareUrl | ||
| 106 | }, | ||
| 107 | transaction: t | ||
| 108 | }) | ||
| 109 | |||
| 110 | return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) | ||
| 111 | } | ||
| 112 | |||
| 113 | async function undoShareByVideoChannel (video: MVideo, oldVideoChannel: MChannelActorLight, t: Transaction) { | ||
| 114 | // Load old share | ||
| 115 | const oldShare = await VideoShareModel.load(oldVideoChannel.actorId, video.id, t) | ||
| 116 | if (!oldShare) return new Error('Cannot find old video channel share ' + oldVideoChannel.actorId + ' for video ' + video.id) | ||
| 117 | |||
| 118 | await sendUndoAnnounce(oldVideoChannel.Actor, oldShare, video, t) | ||
| 119 | await oldShare.destroy({ transaction: t }) | ||
| 120 | } | ||
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts deleted file mode 100644 index 5cdac71bf..000000000 --- a/server/lib/activitypub/url.ts +++ /dev/null | |||
| @@ -1,177 +0,0 @@ | |||
| 1 | import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants' | ||
| 2 | import { | ||
| 3 | MAbuseFull, | ||
| 4 | MAbuseId, | ||
| 5 | MActor, | ||
| 6 | MActorFollow, | ||
| 7 | MActorId, | ||
| 8 | MActorUrl, | ||
| 9 | MCommentId, | ||
| 10 | MLocalVideoViewer, | ||
| 11 | MVideoId, | ||
| 12 | MVideoPlaylistElement, | ||
| 13 | MVideoUrl, | ||
| 14 | MVideoUUID, | ||
| 15 | MVideoWithHost | ||
| 16 | } from '../../types/models' | ||
| 17 | import { MVideoFileVideoUUID } from '../../types/models/video/video-file' | ||
| 18 | import { MVideoPlaylist, MVideoPlaylistUUID } from '../../types/models/video/video-playlist' | ||
| 19 | import { MStreamingPlaylist } from '../../types/models/video/video-streaming-playlist' | ||
| 20 | |||
| 21 | function getLocalVideoActivityPubUrl (video: MVideoUUID) { | ||
| 22 | return WEBSERVER.URL + '/videos/watch/' + video.uuid | ||
| 23 | } | ||
| 24 | |||
| 25 | function getLocalVideoPlaylistActivityPubUrl (videoPlaylist: MVideoPlaylist) { | ||
| 26 | return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid | ||
| 27 | } | ||
| 28 | |||
| 29 | function getLocalVideoPlaylistElementActivityPubUrl (videoPlaylist: MVideoPlaylistUUID, videoPlaylistElement: MVideoPlaylistElement) { | ||
| 30 | return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid + '/videos/' + videoPlaylistElement.id | ||
| 31 | } | ||
| 32 | |||
| 33 | function getLocalVideoCacheFileActivityPubUrl (videoFile: MVideoFileVideoUUID) { | ||
| 34 | const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' | ||
| 35 | |||
| 36 | return `${WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` | ||
| 37 | } | ||
| 38 | |||
| 39 | function getLocalVideoCacheStreamingPlaylistActivityPubUrl (video: MVideoUUID, playlist: MStreamingPlaylist) { | ||
| 40 | return `${WEBSERVER.URL}/redundancy/streaming-playlists/${playlist.getStringType()}/${video.uuid}` | ||
| 41 | } | ||
| 42 | |||
| 43 | function getLocalVideoCommentActivityPubUrl (video: MVideoUUID, videoComment: MCommentId) { | ||
| 44 | return WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id | ||
| 45 | } | ||
| 46 | |||
| 47 | function getLocalVideoChannelActivityPubUrl (videoChannelName: string) { | ||
| 48 | return WEBSERVER.URL + '/video-channels/' + videoChannelName | ||
| 49 | } | ||
| 50 | |||
| 51 | function getLocalAccountActivityPubUrl (accountName: string) { | ||
| 52 | return WEBSERVER.URL + '/accounts/' + accountName | ||
| 53 | } | ||
| 54 | |||
| 55 | function getLocalAbuseActivityPubUrl (abuse: MAbuseId) { | ||
| 56 | return WEBSERVER.URL + '/admin/abuses/' + abuse.id | ||
| 57 | } | ||
| 58 | |||
| 59 | function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId, viewerIdentifier: string) { | ||
| 60 | return byActor.url + '/views/videos/' + video.id + '/' + viewerIdentifier | ||
| 61 | } | ||
| 62 | |||
| 63 | function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { | ||
| 64 | return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid | ||
| 65 | } | ||
| 66 | |||
| 67 | function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { | ||
| 68 | return byActor.url + '/likes/' + video.id | ||
| 69 | } | ||
| 70 | |||
| 71 | function getVideoDislikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { | ||
| 72 | return byActor.url + '/dislikes/' + video.id | ||
| 73 | } | ||
| 74 | |||
| 75 | function getLocalVideoSharesActivityPubUrl (video: MVideoUrl) { | ||
| 76 | return video.url + '/announces' | ||
| 77 | } | ||
| 78 | |||
| 79 | function getLocalVideoCommentsActivityPubUrl (video: MVideoUrl) { | ||
| 80 | return video.url + '/comments' | ||
| 81 | } | ||
| 82 | |||
| 83 | function getLocalVideoLikesActivityPubUrl (video: MVideoUrl) { | ||
| 84 | return video.url + '/likes' | ||
| 85 | } | ||
| 86 | |||
| 87 | function getLocalVideoDislikesActivityPubUrl (video: MVideoUrl) { | ||
| 88 | return video.url + '/dislikes' | ||
| 89 | } | ||
| 90 | |||
| 91 | function getLocalActorFollowActivityPubUrl (follower: MActor, following: MActorId) { | ||
| 92 | return follower.url + '/follows/' + following.id | ||
| 93 | } | ||
| 94 | |||
| 95 | function getLocalActorFollowAcceptActivityPubUrl (actorFollow: MActorFollow) { | ||
| 96 | return WEBSERVER.URL + '/accepts/follows/' + actorFollow.id | ||
| 97 | } | ||
| 98 | |||
| 99 | function getLocalActorFollowRejectActivityPubUrl () { | ||
| 100 | return WEBSERVER.URL + '/rejects/follows/' + new Date().toISOString() | ||
| 101 | } | ||
| 102 | |||
| 103 | function getLocalVideoAnnounceActivityPubUrl (byActor: MActorId, video: MVideoUrl) { | ||
| 104 | return video.url + '/announces/' + byActor.id | ||
| 105 | } | ||
| 106 | |||
| 107 | function getDeleteActivityPubUrl (originalUrl: string) { | ||
| 108 | return originalUrl + '/delete' | ||
| 109 | } | ||
| 110 | |||
| 111 | function getUpdateActivityPubUrl (originalUrl: string, updatedAt: string) { | ||
| 112 | return originalUrl + '/updates/' + updatedAt | ||
| 113 | } | ||
| 114 | |||
| 115 | function getUndoActivityPubUrl (originalUrl: string) { | ||
| 116 | return originalUrl + '/undo' | ||
| 117 | } | ||
| 118 | |||
| 119 | // --------------------------------------------------------------------------- | ||
| 120 | |||
| 121 | function getAbuseTargetUrl (abuse: MAbuseFull) { | ||
| 122 | return abuse.VideoAbuse?.Video?.url || | ||
| 123 | abuse.VideoCommentAbuse?.VideoComment?.url || | ||
| 124 | abuse.FlaggedAccount.Actor.url | ||
| 125 | } | ||
| 126 | |||
| 127 | // --------------------------------------------------------------------------- | ||
| 128 | |||
| 129 | function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string, scheme?: string) { | ||
| 130 | if (!scheme) scheme = REMOTE_SCHEME.HTTP | ||
| 131 | |||
| 132 | const host = video.VideoChannel.Actor.Server.host | ||
| 133 | |||
| 134 | return scheme + '://' + host + path | ||
| 135 | } | ||
| 136 | |||
| 137 | // --------------------------------------------------------------------------- | ||
| 138 | |||
| 139 | function checkUrlsSameHost (url1: string, url2: string) { | ||
| 140 | const idHost = new URL(url1).host | ||
| 141 | const actorHost = new URL(url2).host | ||
| 142 | |||
| 143 | return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() | ||
| 144 | } | ||
| 145 | |||
| 146 | // --------------------------------------------------------------------------- | ||
| 147 | |||
| 148 | export { | ||
| 149 | getLocalVideoActivityPubUrl, | ||
| 150 | getLocalVideoPlaylistActivityPubUrl, | ||
| 151 | getLocalVideoPlaylistElementActivityPubUrl, | ||
| 152 | getLocalVideoCacheFileActivityPubUrl, | ||
| 153 | getLocalVideoCacheStreamingPlaylistActivityPubUrl, | ||
| 154 | getLocalVideoCommentActivityPubUrl, | ||
| 155 | getLocalVideoChannelActivityPubUrl, | ||
| 156 | getLocalAccountActivityPubUrl, | ||
| 157 | getLocalAbuseActivityPubUrl, | ||
| 158 | getLocalActorFollowActivityPubUrl, | ||
| 159 | getLocalActorFollowAcceptActivityPubUrl, | ||
| 160 | getLocalVideoAnnounceActivityPubUrl, | ||
| 161 | getUpdateActivityPubUrl, | ||
| 162 | getUndoActivityPubUrl, | ||
| 163 | getVideoLikeActivityPubUrlByLocalActor, | ||
| 164 | getLocalVideoViewActivityPubUrl, | ||
| 165 | getVideoDislikeActivityPubUrlByLocalActor, | ||
| 166 | getLocalActorFollowRejectActivityPubUrl, | ||
| 167 | getDeleteActivityPubUrl, | ||
| 168 | getLocalVideoSharesActivityPubUrl, | ||
| 169 | getLocalVideoCommentsActivityPubUrl, | ||
| 170 | getLocalVideoLikesActivityPubUrl, | ||
| 171 | getLocalVideoDislikesActivityPubUrl, | ||
| 172 | getLocalVideoViewerActivityPubUrl, | ||
| 173 | |||
| 174 | getAbuseTargetUrl, | ||
| 175 | checkUrlsSameHost, | ||
| 176 | buildRemoteVideoBaseUrl | ||
| 177 | } | ||
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts deleted file mode 100644 index b861be5bd..000000000 --- a/server/lib/activitypub/video-comments.ts +++ /dev/null | |||
| @@ -1,205 +0,0 @@ | |||
| 1 | import { map } from 'bluebird' | ||
| 2 | |||
| 3 | import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' | ||
| 4 | import { logger } from '../../helpers/logger' | ||
| 5 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | ||
| 6 | import { VideoCommentModel } from '../../models/video/video-comment' | ||
| 7 | import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' | ||
| 8 | import { isRemoteVideoCommentAccepted } from '../moderation' | ||
| 9 | import { Hooks } from '../plugins/hooks' | ||
| 10 | import { fetchAP } from './activity' | ||
| 11 | import { getOrCreateAPActor } from './actors' | ||
| 12 | import { checkUrlsSameHost } from './url' | ||
| 13 | import { getOrCreateAPVideo } from './videos' | ||
| 14 | |||
| 15 | type ResolveThreadParams = { | ||
| 16 | url: string | ||
| 17 | comments?: MCommentOwner[] | ||
| 18 | isVideo?: boolean | ||
| 19 | commentCreated?: boolean | ||
| 20 | } | ||
| 21 | type ResolveThreadResult = Promise<{ video: MVideoAccountLightBlacklistAllFiles, comment: MCommentOwnerVideo, commentCreated: boolean }> | ||
| 22 | |||
| 23 | async function addVideoComments (commentUrls: string[]) { | ||
| 24 | return map(commentUrls, async commentUrl => { | ||
| 25 | try { | ||
| 26 | await resolveThread({ url: commentUrl, isVideo: false }) | ||
| 27 | } catch (err) { | ||
| 28 | logger.warn('Cannot resolve thread %s.', commentUrl, { err }) | ||
| 29 | } | ||
| 30 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
| 31 | } | ||
| 32 | |||
| 33 | async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { | ||
| 34 | const { url, isVideo } = params | ||
| 35 | |||
| 36 | if (params.commentCreated === undefined) params.commentCreated = false | ||
| 37 | if (params.comments === undefined) params.comments = [] | ||
| 38 | |||
| 39 | // If it is not a video, or if we don't know if it's a video, try to get the thread from DB | ||
| 40 | if (isVideo === false || isVideo === undefined) { | ||
| 41 | const result = await resolveCommentFromDB(params) | ||
| 42 | if (result) return result | ||
| 43 | } | ||
| 44 | |||
| 45 | try { | ||
| 46 | // If it is a video, or if we don't know if it's a video | ||
| 47 | if (isVideo === true || isVideo === undefined) { | ||
| 48 | // Keep await so we catch the exception | ||
| 49 | return await tryToResolveThreadFromVideo(params) | ||
| 50 | } | ||
| 51 | } catch (err) { | ||
| 52 | logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) | ||
| 53 | } | ||
| 54 | |||
| 55 | return resolveRemoteParentComment(params) | ||
| 56 | } | ||
| 57 | |||
| 58 | export { | ||
| 59 | addVideoComments, | ||
| 60 | resolveThread | ||
| 61 | } | ||
| 62 | |||
| 63 | // --------------------------------------------------------------------------- | ||
| 64 | |||
| 65 | async function resolveCommentFromDB (params: ResolveThreadParams) { | ||
| 66 | const { url, comments, commentCreated } = params | ||
| 67 | |||
| 68 | const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) | ||
| 69 | if (!commentFromDatabase) return undefined | ||
| 70 | |||
| 71 | let parentComments = comments.concat([ commentFromDatabase ]) | ||
| 72 | |||
| 73 | // Speed up things and resolve directly the thread | ||
| 74 | if (commentFromDatabase.InReplyToVideoComment) { | ||
| 75 | const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') | ||
| 76 | |||
| 77 | parentComments = parentComments.concat(data) | ||
| 78 | } | ||
| 79 | |||
| 80 | return resolveThread({ | ||
| 81 | url: commentFromDatabase.Video.url, | ||
| 82 | comments: parentComments, | ||
| 83 | isVideo: true, | ||
| 84 | commentCreated | ||
| 85 | }) | ||
| 86 | } | ||
| 87 | |||
| 88 | async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { | ||
| 89 | const { url, comments, commentCreated } = params | ||
| 90 | |||
| 91 | // Maybe it's a reply to a video? | ||
| 92 | // If yes, it's done: we resolved all the thread | ||
| 93 | const syncParam = { rates: true, shares: true, comments: false, refreshVideo: false } | ||
| 94 | const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam }) | ||
| 95 | |||
| 96 | if (video.isOwned() && !video.hasPrivacyForFederation()) { | ||
| 97 | throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') | ||
| 98 | } | ||
| 99 | |||
| 100 | let resultComment: MCommentOwnerVideo | ||
| 101 | if (comments.length !== 0) { | ||
| 102 | const firstReply = comments[comments.length - 1] as MCommentOwnerVideo | ||
| 103 | firstReply.inReplyToCommentId = null | ||
| 104 | firstReply.originCommentId = null | ||
| 105 | firstReply.videoId = video.id | ||
| 106 | firstReply.changed('updatedAt', true) | ||
| 107 | firstReply.Video = video | ||
| 108 | |||
| 109 | if (await isRemoteCommentAccepted(firstReply) !== true) { | ||
| 110 | return undefined | ||
| 111 | } | ||
| 112 | |||
| 113 | comments[comments.length - 1] = await firstReply.save() | ||
| 114 | |||
| 115 | for (let i = comments.length - 2; i >= 0; i--) { | ||
| 116 | const comment = comments[i] as MCommentOwnerVideo | ||
| 117 | comment.originCommentId = firstReply.id | ||
| 118 | comment.inReplyToCommentId = comments[i + 1].id | ||
| 119 | comment.videoId = video.id | ||
| 120 | comment.changed('updatedAt', true) | ||
| 121 | comment.Video = video | ||
| 122 | |||
| 123 | if (await isRemoteCommentAccepted(comment) !== true) { | ||
| 124 | return undefined | ||
| 125 | } | ||
| 126 | |||
| 127 | comments[i] = await comment.save() | ||
| 128 | } | ||
| 129 | |||
| 130 | resultComment = comments[0] as MCommentOwnerVideo | ||
| 131 | } | ||
| 132 | |||
| 133 | return { video, comment: resultComment, commentCreated } | ||
| 134 | } | ||
| 135 | |||
| 136 | async function resolveRemoteParentComment (params: ResolveThreadParams) { | ||
| 137 | const { url, comments } = params | ||
| 138 | |||
| 139 | if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { | ||
| 140 | throw new Error('Recursion limit reached when resolving a thread') | ||
| 141 | } | ||
| 142 | |||
| 143 | const { body } = await fetchAP<any>(url) | ||
| 144 | |||
| 145 | if (sanitizeAndCheckVideoCommentObject(body) === false) { | ||
| 146 | throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) | ||
| 147 | } | ||
| 148 | |||
| 149 | const actorUrl = body.attributedTo | ||
| 150 | if (!actorUrl && body.type !== 'Tombstone') throw new Error('Miss attributed to in comment') | ||
| 151 | |||
| 152 | if (actorUrl && checkUrlsSameHost(url, actorUrl) !== true) { | ||
| 153 | throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`) | ||
| 154 | } | ||
| 155 | |||
| 156 | if (checkUrlsSameHost(body.id, url) !== true) { | ||
| 157 | throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`) | ||
| 158 | } | ||
| 159 | |||
| 160 | const actor = actorUrl | ||
| 161 | ? await getOrCreateAPActor(actorUrl, 'all') | ||
| 162 | : null | ||
| 163 | |||
| 164 | const comment = new VideoCommentModel({ | ||
| 165 | url: body.id, | ||
| 166 | text: body.content ? body.content : '', | ||
| 167 | videoId: null, | ||
| 168 | accountId: actor ? actor.Account.id : null, | ||
| 169 | inReplyToCommentId: null, | ||
| 170 | originCommentId: null, | ||
| 171 | createdAt: new Date(body.published), | ||
| 172 | updatedAt: new Date(body.updated), | ||
| 173 | deletedAt: body.deleted ? new Date(body.deleted) : null | ||
| 174 | }) as MCommentOwner | ||
| 175 | comment.Account = actor ? actor.Account : null | ||
| 176 | |||
| 177 | return resolveThread({ | ||
| 178 | url: body.inReplyTo, | ||
| 179 | comments: comments.concat([ comment ]), | ||
| 180 | commentCreated: true | ||
| 181 | }) | ||
| 182 | } | ||
| 183 | |||
| 184 | async function isRemoteCommentAccepted (comment: MComment) { | ||
| 185 | // Already created | ||
| 186 | if (comment.id) return true | ||
| 187 | |||
| 188 | const acceptParameters = { | ||
| 189 | comment | ||
| 190 | } | ||
| 191 | |||
| 192 | const acceptedResult = await Hooks.wrapFun( | ||
| 193 | isRemoteVideoCommentAccepted, | ||
| 194 | acceptParameters, | ||
| 195 | 'filter:activity-pub.remote-video-comment.create.accept.result' | ||
| 196 | ) | ||
| 197 | |||
| 198 | if (!acceptedResult || acceptedResult.accepted !== true) { | ||
| 199 | logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters }) | ||
| 200 | |||
| 201 | return false | ||
| 202 | } | ||
| 203 | |||
| 204 | return true | ||
| 205 | } | ||
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts deleted file mode 100644 index 2e7920f4e..000000000 --- a/server/lib/activitypub/video-rates.ts +++ /dev/null | |||
| @@ -1,59 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { VideoRateType } from '../../../shared/models/videos' | ||
| 3 | import { MAccountActor, MActorUrl, MVideoAccountLight, MVideoFullLight, MVideoId } from '../../types/models' | ||
| 4 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' | ||
| 5 | import { sendDislike } from './send/send-dislike' | ||
| 6 | import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' | ||
| 7 | import { federateVideoIfNeeded } from './videos' | ||
| 8 | |||
| 9 | async function sendVideoRateChange ( | ||
| 10 | account: MAccountActor, | ||
| 11 | video: MVideoFullLight, | ||
| 12 | likes: number, | ||
| 13 | dislikes: number, | ||
| 14 | t: Transaction | ||
| 15 | ) { | ||
| 16 | if (video.isOwned()) return federateVideoIfNeeded(video, false, t) | ||
| 17 | |||
| 18 | return sendVideoRateChangeToOrigin(account, video, likes, dislikes, t) | ||
| 19 | } | ||
| 20 | |||
| 21 | function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVideoId) { | ||
| 22 | return rateType === 'like' | ||
| 23 | ? getVideoLikeActivityPubUrlByLocalActor(actor, video) | ||
| 24 | : getVideoDislikeActivityPubUrlByLocalActor(actor, video) | ||
| 25 | } | ||
| 26 | |||
| 27 | // --------------------------------------------------------------------------- | ||
| 28 | |||
| 29 | export { | ||
| 30 | getLocalRateUrl, | ||
| 31 | sendVideoRateChange | ||
| 32 | } | ||
| 33 | |||
| 34 | // --------------------------------------------------------------------------- | ||
| 35 | |||
| 36 | async function sendVideoRateChangeToOrigin ( | ||
| 37 | account: MAccountActor, | ||
| 38 | video: MVideoAccountLight, | ||
| 39 | likes: number, | ||
| 40 | dislikes: number, | ||
| 41 | t: Transaction | ||
| 42 | ) { | ||
| 43 | // Local video, we don't need to send like | ||
| 44 | if (video.isOwned()) return | ||
| 45 | |||
| 46 | const actor = account.Actor | ||
| 47 | |||
| 48 | // Keep the order: first we undo and then we create | ||
| 49 | |||
| 50 | // Undo Like | ||
| 51 | if (likes < 0) await sendUndoLike(actor, video, t) | ||
| 52 | // Undo Dislike | ||
| 53 | if (dislikes < 0) await sendUndoDislike(actor, video, t) | ||
| 54 | |||
| 55 | // Like | ||
| 56 | if (likes > 0) await sendLike(actor, video, t) | ||
| 57 | // Dislike | ||
| 58 | if (dislikes > 0) await sendDislike(actor, video, t) | ||
| 59 | } | ||
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts deleted file mode 100644 index d7e251153..000000000 --- a/server/lib/activitypub/videos/federate.ts +++ /dev/null | |||
| @@ -1,29 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize/types' | ||
| 2 | import { MVideoAP, MVideoAPLight } from '@server/types/models' | ||
| 3 | import { sendCreateVideo, sendUpdateVideo } from '../send' | ||
| 4 | import { shareVideoByServerAndChannel } from '../share' | ||
| 5 | |||
| 6 | async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) { | ||
| 7 | const video = videoArg as MVideoAP | ||
| 8 | |||
| 9 | if ( | ||
| 10 | // Check this is not a blacklisted video, or unfederated blacklisted video | ||
| 11 | (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) && | ||
| 12 | // Check the video is public/unlisted and published | ||
| 13 | video.hasPrivacyForFederation() && video.hasStateForFederation() | ||
| 14 | ) { | ||
| 15 | const video = await videoArg.lightAPToFullAP(transaction) | ||
| 16 | |||
| 17 | if (isNewVideo) { | ||
| 18 | // Now we'll add the video's meta data to our followers | ||
| 19 | await sendCreateVideo(video, transaction) | ||
| 20 | await shareVideoByServerAndChannel(video, transaction) | ||
| 21 | } else { | ||
| 22 | await sendUpdateVideo(video, transaction) | ||
| 23 | } | ||
| 24 | } | ||
| 25 | } | ||
| 26 | |||
| 27 | export { | ||
| 28 | federateVideoIfNeeded | ||
| 29 | } | ||
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts deleted file mode 100644 index 288c506ee..000000000 --- a/server/lib/activitypub/videos/get.ts +++ /dev/null | |||
| @@ -1,116 +0,0 @@ | |||
| 1 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { JobQueue } from '@server/lib/job-queue' | ||
| 4 | import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders' | ||
| 5 | import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | ||
| 6 | import { APObjectId } from '@shared/models' | ||
| 7 | import { getAPId } from '../activity' | ||
| 8 | import { refreshVideoIfNeeded } from './refresh' | ||
| 9 | import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | ||
| 10 | |||
| 11 | type GetVideoResult <T> = Promise<{ | ||
| 12 | video: T | ||
| 13 | created: boolean | ||
| 14 | autoBlacklisted?: boolean | ||
| 15 | }> | ||
| 16 | |||
| 17 | type GetVideoParamAll = { | ||
| 18 | videoObject: APObjectId | ||
| 19 | syncParam?: SyncParam | ||
| 20 | fetchType?: 'all' | ||
| 21 | allowRefresh?: boolean | ||
| 22 | } | ||
| 23 | |||
| 24 | type GetVideoParamImmutable = { | ||
| 25 | videoObject: APObjectId | ||
| 26 | syncParam?: SyncParam | ||
| 27 | fetchType: 'only-immutable-attributes' | ||
| 28 | allowRefresh: false | ||
| 29 | } | ||
| 30 | |||
| 31 | type GetVideoParamOther = { | ||
| 32 | videoObject: APObjectId | ||
| 33 | syncParam?: SyncParam | ||
| 34 | fetchType?: 'all' | 'only-video' | ||
| 35 | allowRefresh?: boolean | ||
| 36 | } | ||
| 37 | |||
| 38 | function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles> | ||
| 39 | function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable> | ||
| 40 | function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> | ||
| 41 | |||
| 42 | async function getOrCreateAPVideo ( | ||
| 43 | options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther | ||
| 44 | ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
| 45 | // Default params | ||
| 46 | const syncParam = options.syncParam || { rates: true, shares: true, comments: true, refreshVideo: false } | ||
| 47 | const fetchType = options.fetchType || 'all' | ||
| 48 | const allowRefresh = options.allowRefresh !== false | ||
| 49 | |||
| 50 | // Get video url | ||
| 51 | const videoUrl = getAPId(options.videoObject) | ||
| 52 | let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) | ||
| 53 | |||
| 54 | if (videoFromDatabase) { | ||
| 55 | if (allowRefresh === true) { | ||
| 56 | // Typings ensure allowRefresh === false in only-immutable-attributes fetch type | ||
| 57 | videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) | ||
| 58 | } | ||
| 59 | |||
| 60 | return { video: videoFromDatabase, created: false } | ||
| 61 | } | ||
| 62 | |||
| 63 | const { videoObject } = await fetchRemoteVideo(videoUrl) | ||
| 64 | if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
| 65 | |||
| 66 | // videoUrl is just an alias/rediraction, so process object id instead | ||
| 67 | if (videoObject.id !== videoUrl) return getOrCreateAPVideo({ ...options, fetchType: 'all', videoObject }) | ||
| 68 | |||
| 69 | try { | ||
| 70 | const creator = new APVideoCreator(videoObject) | ||
| 71 | const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator)) | ||
| 72 | |||
| 73 | await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) | ||
| 74 | |||
| 75 | return { video: videoCreated, created: true, autoBlacklisted } | ||
| 76 | } catch (err) { | ||
| 77 | // Maybe a concurrent getOrCreateAPVideo call created this video | ||
| 78 | if (err.name === 'SequelizeUniqueConstraintError') { | ||
| 79 | const alreadyCreatedVideo = await loadVideoByUrl(videoUrl, fetchType) | ||
| 80 | if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } | ||
| 81 | |||
| 82 | logger.error('Cannot create video %s because of SequelizeUniqueConstraintError error, but cannot find it in database.', videoUrl) | ||
| 83 | } | ||
| 84 | |||
| 85 | throw err | ||
| 86 | } | ||
| 87 | } | ||
| 88 | |||
| 89 | // --------------------------------------------------------------------------- | ||
| 90 | |||
| 91 | export { | ||
| 92 | getOrCreateAPVideo | ||
| 93 | } | ||
| 94 | |||
| 95 | // --------------------------------------------------------------------------- | ||
| 96 | |||
| 97 | async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) { | ||
| 98 | if (!video.isOutdated()) return video | ||
| 99 | |||
| 100 | const refreshOptions = { | ||
| 101 | video, | ||
| 102 | fetchedType: fetchType, | ||
| 103 | syncParam | ||
| 104 | } | ||
| 105 | |||
| 106 | if (syncParam.refreshVideo === true) { | ||
| 107 | return refreshVideoIfNeeded(refreshOptions) | ||
| 108 | } | ||
| 109 | |||
| 110 | await JobQueue.Instance.createJob({ | ||
| 111 | type: 'activitypub-refresher', | ||
| 112 | payload: { type: 'video', url: video.url } | ||
| 113 | }) | ||
| 114 | |||
| 115 | return video | ||
| 116 | } | ||
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts deleted file mode 100644 index b22062598..000000000 --- a/server/lib/activitypub/videos/index.ts +++ /dev/null | |||
| @@ -1,4 +0,0 @@ | |||
| 1 | export * from './federate' | ||
| 2 | export * from './get' | ||
| 3 | export * from './refresh' | ||
| 4 | export * from './updater' | ||
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts deleted file mode 100644 index 9f952a218..000000000 --- a/server/lib/activitypub/videos/refresh.ts +++ /dev/null | |||
| @@ -1,68 +0,0 @@ | |||
| 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 2 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
| 3 | import { VideoLoadByUrlType } from '@server/lib/model-loaders' | ||
| 4 | import { VideoModel } from '@server/models/video/video' | ||
| 5 | import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models' | ||
| 6 | import { HttpStatusCode } from '@shared/models' | ||
| 7 | import { ActorFollowHealthCache } from '../../actor-follow-health-cache' | ||
| 8 | import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | ||
| 9 | import { APVideoUpdater } from './updater' | ||
| 10 | |||
| 11 | async function refreshVideoIfNeeded (options: { | ||
| 12 | video: MVideoThumbnail | ||
| 13 | fetchedType: VideoLoadByUrlType | ||
| 14 | syncParam: SyncParam | ||
| 15 | }): Promise<MVideoThumbnail> { | ||
| 16 | if (!options.video.isOutdated()) return options.video | ||
| 17 | |||
| 18 | // We need more attributes if the argument video was fetched with not enough joints | ||
| 19 | const video = options.fetchedType === 'all' | ||
| 20 | ? options.video as MVideoAccountLightBlacklistAllFiles | ||
| 21 | : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
| 22 | |||
| 23 | const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url) | ||
| 24 | |||
| 25 | logger.info('Refreshing video %s.', video.url, lTags()) | ||
| 26 | |||
| 27 | try { | ||
| 28 | const { videoObject } = await fetchRemoteVideo(video.url) | ||
| 29 | |||
| 30 | if (videoObject === undefined) { | ||
| 31 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags()) | ||
| 32 | |||
| 33 | await video.setAsRefreshed() | ||
| 34 | return video | ||
| 35 | } | ||
| 36 | |||
| 37 | const videoUpdater = new APVideoUpdater(videoObject, video) | ||
| 38 | await videoUpdater.update() | ||
| 39 | |||
| 40 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
| 41 | |||
| 42 | ActorFollowHealthCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) | ||
| 43 | |||
| 44 | return video | ||
| 45 | } catch (err) { | ||
| 46 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
| 47 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags()) | ||
| 48 | |||
| 49 | // Video does not exist anymore | ||
| 50 | await video.destroy() | ||
| 51 | return undefined | ||
| 52 | } | ||
| 53 | |||
| 54 | logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags() }) | ||
| 55 | |||
| 56 | ActorFollowHealthCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | ||
| 57 | |||
| 58 | // Don't refresh in loop | ||
| 59 | await video.setAsRefreshed() | ||
| 60 | return video | ||
| 61 | } | ||
| 62 | } | ||
| 63 | |||
| 64 | // --------------------------------------------------------------------------- | ||
| 65 | |||
| 66 | export { | ||
| 67 | refreshVideoIfNeeded | ||
| 68 | } | ||
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts deleted file mode 100644 index 98c2f58eb..000000000 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ /dev/null | |||
| @@ -1,190 +0,0 @@ | |||
| 1 | import { CreationAttributes, Transaction } from 'sequelize/types' | ||
| 2 | import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' | ||
| 3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' | ||
| 4 | import { updateRemoteVideoThumbnail } from '@server/lib/thumbnail' | ||
| 5 | import { setVideoTags } from '@server/lib/video' | ||
| 6 | import { StoryboardModel } from '@server/models/video/storyboard' | ||
| 7 | import { VideoCaptionModel } from '@server/models/video/video-caption' | ||
| 8 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 9 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
| 10 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
| 11 | import { | ||
| 12 | MStreamingPlaylistFiles, | ||
| 13 | MStreamingPlaylistFilesVideo, | ||
| 14 | MVideoCaption, | ||
| 15 | MVideoFile, | ||
| 16 | MVideoFullLight, | ||
| 17 | MVideoThumbnail | ||
| 18 | } from '@server/types/models' | ||
| 19 | import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' | ||
| 20 | import { findOwner, getOrCreateAPActor } from '../../actors' | ||
| 21 | import { | ||
| 22 | getCaptionAttributesFromObject, | ||
| 23 | getFileAttributesFromUrl, | ||
| 24 | getLiveAttributesFromObject, | ||
| 25 | getPreviewFromIcons, | ||
| 26 | getStoryboardAttributeFromObject, | ||
| 27 | getStreamingPlaylistAttributesFromObject, | ||
| 28 | getTagsFromObject, | ||
| 29 | getThumbnailFromIcons | ||
| 30 | } from './object-to-model-attributes' | ||
| 31 | import { getTrackerUrls, setVideoTrackers } from './trackers' | ||
| 32 | |||
| 33 | export abstract class APVideoAbstractBuilder { | ||
| 34 | protected abstract videoObject: VideoObject | ||
| 35 | protected abstract lTags: LoggerTagsFn | ||
| 36 | |||
| 37 | protected async getOrCreateVideoChannelFromVideoObject () { | ||
| 38 | const channel = await findOwner(this.videoObject.id, this.videoObject.attributedTo, 'Group') | ||
| 39 | if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url) | ||
| 40 | |||
| 41 | return getOrCreateAPActor(channel.id, 'all') | ||
| 42 | } | ||
| 43 | |||
| 44 | protected async setThumbnail (video: MVideoThumbnail, t?: Transaction) { | ||
| 45 | const miniatureIcon = getThumbnailFromIcons(this.videoObject) | ||
| 46 | if (!miniatureIcon) { | ||
| 47 | logger.warn('Cannot find thumbnail in video object', { object: this.videoObject }) | ||
| 48 | return undefined | ||
| 49 | } | ||
| 50 | |||
| 51 | const miniatureModel = updateRemoteVideoThumbnail({ | ||
| 52 | fileUrl: miniatureIcon.url, | ||
| 53 | video, | ||
| 54 | type: ThumbnailType.MINIATURE, | ||
| 55 | size: miniatureIcon, | ||
| 56 | onDisk: false // Lazy download remote thumbnails | ||
| 57 | }) | ||
| 58 | |||
| 59 | await video.addAndSaveThumbnail(miniatureModel, t) | ||
| 60 | } | ||
| 61 | |||
| 62 | protected async setPreview (video: MVideoFullLight, t?: Transaction) { | ||
| 63 | const previewIcon = getPreviewFromIcons(this.videoObject) | ||
| 64 | if (!previewIcon) return | ||
| 65 | |||
| 66 | const previewModel = updateRemoteVideoThumbnail({ | ||
| 67 | fileUrl: previewIcon.url, | ||
| 68 | video, | ||
| 69 | type: ThumbnailType.PREVIEW, | ||
| 70 | size: previewIcon, | ||
| 71 | onDisk: false // Lazy download remote previews | ||
| 72 | }) | ||
| 73 | |||
| 74 | await video.addAndSaveThumbnail(previewModel, t) | ||
| 75 | } | ||
| 76 | |||
| 77 | protected async setTags (video: MVideoFullLight, t: Transaction) { | ||
| 78 | const tags = getTagsFromObject(this.videoObject) | ||
| 79 | await setVideoTags({ video, tags, transaction: t }) | ||
| 80 | } | ||
| 81 | |||
| 82 | protected async setTrackers (video: MVideoFullLight, t: Transaction) { | ||
| 83 | const trackers = getTrackerUrls(this.videoObject, video) | ||
| 84 | await setVideoTrackers({ video, trackers, transaction: t }) | ||
| 85 | } | ||
| 86 | |||
| 87 | protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) { | ||
| 88 | const existingCaptions = await VideoCaptionModel.listVideoCaptions(video.id, t) | ||
| 89 | |||
| 90 | let captionsToCreate = getCaptionAttributesFromObject(video, this.videoObject) | ||
| 91 | .map(a => new VideoCaptionModel(a) as MVideoCaption) | ||
| 92 | |||
| 93 | for (const existingCaption of existingCaptions) { | ||
| 94 | // Only keep captions that do not already exist | ||
| 95 | const filtered = captionsToCreate.filter(c => !c.isEqual(existingCaption)) | ||
| 96 | |||
| 97 | // This caption already exists, we don't need to destroy and create it | ||
| 98 | if (filtered.length !== captionsToCreate.length) { | ||
| 99 | captionsToCreate = filtered | ||
| 100 | continue | ||
| 101 | } | ||
| 102 | |||
| 103 | // Destroy this caption that does not exist anymore | ||
| 104 | await existingCaption.destroy({ transaction: t }) | ||
| 105 | } | ||
| 106 | |||
| 107 | for (const captionToCreate of captionsToCreate) { | ||
| 108 | await captionToCreate.save({ transaction: t }) | ||
| 109 | } | ||
| 110 | } | ||
| 111 | |||
| 112 | protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) { | ||
| 113 | const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t) | ||
| 114 | if (existingStoryboard) await existingStoryboard.destroy({ transaction: t }) | ||
| 115 | |||
| 116 | const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject) | ||
| 117 | if (!storyboardAttributes) return | ||
| 118 | |||
| 119 | return StoryboardModel.create(storyboardAttributes, { transaction: t }) | ||
| 120 | } | ||
| 121 | |||
| 122 | protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { | ||
| 123 | const attributes = getLiveAttributesFromObject(video, this.videoObject) | ||
| 124 | const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) | ||
| 125 | |||
| 126 | video.VideoLive = videoLive | ||
| 127 | } | ||
| 128 | |||
| 129 | protected async setWebVideoFiles (video: MVideoFullLight, t: Transaction) { | ||
| 130 | const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url) | ||
| 131 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) | ||
| 132 | |||
| 133 | // Remove video files that do not exist anymore | ||
| 134 | await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t) | ||
| 135 | |||
| 136 | // Update or add other one | ||
| 137 | const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) | ||
| 138 | video.VideoFiles = await Promise.all(upsertTasks) | ||
| 139 | } | ||
| 140 | |||
| 141 | protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) { | ||
| 142 | const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject) | ||
| 143 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
| 144 | |||
| 145 | // Remove video playlists that do not exist anymore | ||
| 146 | await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) | ||
| 147 | |||
| 148 | const oldPlaylists = video.VideoStreamingPlaylists | ||
| 149 | video.VideoStreamingPlaylists = [] | ||
| 150 | |||
| 151 | for (const playlistAttributes of streamingPlaylistAttributes) { | ||
| 152 | const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) | ||
| 153 | streamingPlaylistModel.Video = video | ||
| 154 | |||
| 155 | await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t) | ||
| 156 | |||
| 157 | video.VideoStreamingPlaylists.push(streamingPlaylistModel) | ||
| 158 | } | ||
| 159 | } | ||
| 160 | |||
| 161 | private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes<VideoStreamingPlaylistModel>, t: Transaction) { | ||
| 162 | const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) | ||
| 163 | |||
| 164 | return streamingPlaylist as MStreamingPlaylistFilesVideo | ||
| 165 | } | ||
| 166 | |||
| 167 | private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType) { | ||
| 168 | const playlist = oldPlaylists.find(s => s.type === type) | ||
| 169 | if (!playlist) return [] | ||
| 170 | |||
| 171 | return playlist.VideoFiles | ||
| 172 | } | ||
| 173 | |||
| 174 | private async setStreamingPlaylistFiles ( | ||
| 175 | oldPlaylists: MStreamingPlaylistFiles[], | ||
| 176 | playlistModel: MStreamingPlaylistFilesVideo, | ||
| 177 | tagObjects: ActivityTagObject[], | ||
| 178 | t: Transaction | ||
| 179 | ) { | ||
| 180 | const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type) | ||
| 181 | |||
| 182 | const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) | ||
| 183 | |||
| 184 | await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t) | ||
| 185 | |||
| 186 | // Update or add other one | ||
| 187 | const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) | ||
| 188 | playlistModel.VideoFiles = await Promise.all(upsertTasks) | ||
| 189 | } | ||
| 190 | } | ||
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts deleted file mode 100644 index e44fd0d52..000000000 --- a/server/lib/activitypub/videos/shared/creator.ts +++ /dev/null | |||
| @@ -1,65 +0,0 @@ | |||
| 1 | |||
| 2 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
| 3 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 4 | import { Hooks } from '@server/lib/plugins/hooks' | ||
| 5 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | ||
| 6 | import { VideoModel } from '@server/models/video/video' | ||
| 7 | import { MVideoFullLight, MVideoThumbnail } from '@server/types/models' | ||
| 8 | import { VideoObject } from '@shared/models' | ||
| 9 | import { APVideoAbstractBuilder } from './abstract-builder' | ||
| 10 | import { getVideoAttributesFromObject } from './object-to-model-attributes' | ||
| 11 | |||
| 12 | export class APVideoCreator extends APVideoAbstractBuilder { | ||
| 13 | protected lTags: LoggerTagsFn | ||
| 14 | |||
| 15 | constructor (protected readonly videoObject: VideoObject) { | ||
| 16 | super() | ||
| 17 | |||
| 18 | this.lTags = loggerTagsFactory('ap', 'video', 'create', this.videoObject.uuid, this.videoObject.id) | ||
| 19 | } | ||
| 20 | |||
| 21 | async create () { | ||
| 22 | logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags()) | ||
| 23 | |||
| 24 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() | ||
| 25 | const channel = channelActor.VideoChannel | ||
| 26 | |||
| 27 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) | ||
| 28 | const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail | ||
| 29 | |||
| 30 | const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { | ||
| 31 | const videoCreated = await video.save({ transaction: t }) as MVideoFullLight | ||
| 32 | videoCreated.VideoChannel = channel | ||
| 33 | |||
| 34 | await this.setThumbnail(videoCreated, t) | ||
| 35 | await this.setPreview(videoCreated, t) | ||
| 36 | await this.setWebVideoFiles(videoCreated, t) | ||
| 37 | await this.setStreamingPlaylists(videoCreated, t) | ||
| 38 | await this.setTags(videoCreated, t) | ||
| 39 | await this.setTrackers(videoCreated, t) | ||
| 40 | await this.insertOrReplaceCaptions(videoCreated, t) | ||
| 41 | await this.insertOrReplaceLive(videoCreated, t) | ||
| 42 | await this.insertOrReplaceStoryboard(videoCreated, t) | ||
| 43 | |||
| 44 | // We added a video in this channel, set it as updated | ||
| 45 | await channel.setAsUpdated(t) | ||
| 46 | |||
| 47 | const autoBlacklisted = await autoBlacklistVideoIfNeeded({ | ||
| 48 | video: videoCreated, | ||
| 49 | user: undefined, | ||
| 50 | isRemote: true, | ||
| 51 | isNew: true, | ||
| 52 | isNewFile: true, | ||
| 53 | transaction: t | ||
| 54 | }) | ||
| 55 | |||
| 56 | logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) | ||
| 57 | |||
| 58 | Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) | ||
| 59 | |||
| 60 | return { autoBlacklisted, videoCreated } | ||
| 61 | }) | ||
| 62 | |||
| 63 | return { autoBlacklisted, videoCreated } | ||
| 64 | } | ||
| 65 | } | ||
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts deleted file mode 100644 index 951403493..000000000 --- a/server/lib/activitypub/videos/shared/index.ts +++ /dev/null | |||
| @@ -1,6 +0,0 @@ | |||
| 1 | export * from './abstract-builder' | ||
| 2 | export * from './creator' | ||
| 3 | export * from './object-to-model-attributes' | ||
| 4 | export * from './trackers' | ||
| 5 | export * from './url-to-object' | ||
| 6 | export * from './video-sync-attributes' | ||
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts deleted file mode 100644 index 6cbe72e27..000000000 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ /dev/null | |||
| @@ -1,285 +0,0 @@ | |||
| 1 | import { maxBy, minBy } from 'lodash' | ||
| 2 | import { decode as magnetUriDecode } from 'magnet-uri' | ||
| 3 | import { basename, extname } from 'path' | ||
| 4 | import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
| 5 | import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' | ||
| 6 | import { logger } from '@server/helpers/logger' | ||
| 7 | import { getExtFromMimetype } from '@server/helpers/video' | ||
| 8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' | ||
| 9 | import { generateTorrentFileName } from '@server/lib/paths' | ||
| 10 | import { VideoCaptionModel } from '@server/models/video/video-caption' | ||
| 11 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 12 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
| 13 | import { FilteredModelAttributes } from '@server/types' | ||
| 14 | import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoId } from '@server/types/models' | ||
| 15 | import { | ||
| 16 | ActivityHashTagObject, | ||
| 17 | ActivityMagnetUrlObject, | ||
| 18 | ActivityPlaylistSegmentHashesObject, | ||
| 19 | ActivityPlaylistUrlObject, | ||
| 20 | ActivityTagObject, | ||
| 21 | ActivityUrlObject, | ||
| 22 | ActivityVideoUrlObject, | ||
| 23 | VideoObject, | ||
| 24 | VideoPrivacy, | ||
| 25 | VideoStreamingPlaylistType | ||
| 26 | } from '@shared/models' | ||
| 27 | import { getDurationFromActivityStream } from '../../activity' | ||
| 28 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
| 29 | import { generateImageFilename } from '@server/helpers/image-utils' | ||
| 30 | import { arrayify } from '@shared/core-utils' | ||
| 31 | |||
| 32 | function getThumbnailFromIcons (videoObject: VideoObject) { | ||
| 33 | let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) | ||
| 34 | // Fallback if there are not valid icons | ||
| 35 | if (validIcons.length === 0) validIcons = videoObject.icon | ||
| 36 | |||
| 37 | return minBy(validIcons, 'width') | ||
| 38 | } | ||
| 39 | |||
| 40 | function getPreviewFromIcons (videoObject: VideoObject) { | ||
| 41 | const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth) | ||
| 42 | |||
| 43 | return maxBy(validIcons, 'width') | ||
| 44 | } | ||
| 45 | |||
| 46 | function getTagsFromObject (videoObject: VideoObject) { | ||
| 47 | return videoObject.tag | ||
| 48 | .filter(isAPHashTagObject) | ||
| 49 | .map(t => t.name) | ||
| 50 | } | ||
| 51 | |||
| 52 | function getFileAttributesFromUrl ( | ||
| 53 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, | ||
| 54 | urls: (ActivityTagObject | ActivityUrlObject)[] | ||
| 55 | ) { | ||
| 56 | const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
| 57 | |||
| 58 | if (fileUrls.length === 0) return [] | ||
| 59 | |||
| 60 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] | ||
| 61 | for (const fileUrl of fileUrls) { | ||
| 62 | // Fetch associated magnet uri | ||
| 63 | const magnet = urls.filter(isAPMagnetUrlObject) | ||
| 64 | .find(u => u.height === fileUrl.height) | ||
| 65 | |||
| 66 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
| 67 | |||
| 68 | const parsed = magnetUriDecode(magnet.href) | ||
| 69 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
| 70 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
| 71 | } | ||
| 72 | |||
| 73 | const torrentUrl = Array.isArray(parsed.xs) | ||
| 74 | ? parsed.xs[0] | ||
| 75 | : parsed.xs | ||
| 76 | |||
| 77 | // Fetch associated metadata url, if any | ||
| 78 | const metadata = urls.filter(isAPVideoFileUrlMetadataObject) | ||
| 79 | .find(u => { | ||
| 80 | return u.height === fileUrl.height && | ||
| 81 | u.fps === fileUrl.fps && | ||
| 82 | u.rel.includes(fileUrl.mediaType) | ||
| 83 | }) | ||
| 84 | |||
| 85 | const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) | ||
| 86 | const resolution = fileUrl.height | ||
| 87 | const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id | ||
| 88 | const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null | ||
| 89 | |||
| 90 | const attribute = { | ||
| 91 | extname, | ||
| 92 | infoHash: parsed.infoHash, | ||
| 93 | resolution, | ||
| 94 | size: fileUrl.size, | ||
| 95 | fps: fileUrl.fps || -1, | ||
| 96 | metadataUrl: metadata?.href, | ||
| 97 | |||
| 98 | // Use the name of the remote file because we don't proxify video file requests | ||
| 99 | filename: basename(fileUrl.href), | ||
| 100 | fileUrl: fileUrl.href, | ||
| 101 | |||
| 102 | torrentUrl, | ||
| 103 | // Use our own torrent name since we proxify torrent requests | ||
| 104 | torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), | ||
| 105 | |||
| 106 | // This is a video file owned by a video or by a streaming playlist | ||
| 107 | videoId, | ||
| 108 | videoStreamingPlaylistId | ||
| 109 | } | ||
| 110 | |||
| 111 | attributes.push(attribute) | ||
| 112 | } | ||
| 113 | |||
| 114 | return attributes | ||
| 115 | } | ||
| 116 | |||
| 117 | function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject) { | ||
| 118 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
| 119 | if (playlistUrls.length === 0) return [] | ||
| 120 | |||
| 121 | const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = [] | ||
| 122 | for (const playlistUrlObject of playlistUrls) { | ||
| 123 | const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject) | ||
| 124 | |||
| 125 | const files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
| 126 | |||
| 127 | if (!segmentsSha256UrlObject) { | ||
| 128 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
| 129 | continue | ||
| 130 | } | ||
| 131 | |||
| 132 | const attribute = { | ||
| 133 | type: VideoStreamingPlaylistType.HLS, | ||
| 134 | |||
| 135 | playlistFilename: basename(playlistUrlObject.href), | ||
| 136 | playlistUrl: playlistUrlObject.href, | ||
| 137 | |||
| 138 | segmentsSha256Filename: basename(segmentsSha256UrlObject.href), | ||
| 139 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
| 140 | |||
| 141 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), | ||
| 142 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
| 143 | videoId: video.id, | ||
| 144 | |||
| 145 | tagAPObject: playlistUrlObject.tag | ||
| 146 | } | ||
| 147 | |||
| 148 | attributes.push(attribute) | ||
| 149 | } | ||
| 150 | |||
| 151 | return attributes | ||
| 152 | } | ||
| 153 | |||
| 154 | function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) { | ||
| 155 | return { | ||
| 156 | saveReplay: videoObject.liveSaveReplay, | ||
| 157 | permanentLive: videoObject.permanentLive, | ||
| 158 | latencyMode: videoObject.latencyMode, | ||
| 159 | videoId: video.id | ||
| 160 | } | ||
| 161 | } | ||
| 162 | |||
| 163 | function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) { | ||
| 164 | return videoObject.subtitleLanguage.map(c => ({ | ||
| 165 | videoId: video.id, | ||
| 166 | filename: VideoCaptionModel.generateCaptionName(c.identifier), | ||
| 167 | language: c.identifier, | ||
| 168 | fileUrl: c.url | ||
| 169 | })) | ||
| 170 | } | ||
| 171 | |||
| 172 | function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) { | ||
| 173 | if (!isArray(videoObject.preview)) return undefined | ||
| 174 | |||
| 175 | const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard')) | ||
| 176 | if (!storyboard) return undefined | ||
| 177 | |||
| 178 | const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg') | ||
| 179 | |||
| 180 | return { | ||
| 181 | filename: generateImageFilename(extname(url.href)), | ||
| 182 | totalHeight: url.height, | ||
| 183 | totalWidth: url.width, | ||
| 184 | spriteHeight: url.tileHeight, | ||
| 185 | spriteWidth: url.tileWidth, | ||
| 186 | spriteDuration: getDurationFromActivityStream(url.tileDuration), | ||
| 187 | fileUrl: url.href, | ||
| 188 | videoId: video.id | ||
| 189 | } | ||
| 190 | } | ||
| 191 | |||
| 192 | function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { | ||
| 193 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | ||
| 194 | ? VideoPrivacy.PUBLIC | ||
| 195 | : VideoPrivacy.UNLISTED | ||
| 196 | |||
| 197 | const language = videoObject.language?.identifier | ||
| 198 | |||
| 199 | const category = videoObject.category | ||
| 200 | ? parseInt(videoObject.category.identifier, 10) | ||
| 201 | : undefined | ||
| 202 | |||
| 203 | const licence = videoObject.licence | ||
| 204 | ? parseInt(videoObject.licence.identifier, 10) | ||
| 205 | : undefined | ||
| 206 | |||
| 207 | const description = videoObject.content || null | ||
| 208 | const support = videoObject.support || null | ||
| 209 | |||
| 210 | return { | ||
| 211 | name: videoObject.name, | ||
| 212 | uuid: videoObject.uuid, | ||
| 213 | url: videoObject.id, | ||
| 214 | category, | ||
| 215 | licence, | ||
| 216 | language, | ||
| 217 | description, | ||
| 218 | support, | ||
| 219 | nsfw: videoObject.sensitive, | ||
| 220 | commentsEnabled: videoObject.commentsEnabled, | ||
| 221 | downloadEnabled: videoObject.downloadEnabled, | ||
| 222 | waitTranscoding: videoObject.waitTranscoding, | ||
| 223 | isLive: videoObject.isLiveBroadcast, | ||
| 224 | state: videoObject.state, | ||
| 225 | channelId: videoChannel.id, | ||
| 226 | duration: getDurationFromActivityStream(videoObject.duration), | ||
| 227 | createdAt: new Date(videoObject.published), | ||
| 228 | publishedAt: new Date(videoObject.published), | ||
| 229 | |||
| 230 | originallyPublishedAt: videoObject.originallyPublishedAt | ||
| 231 | ? new Date(videoObject.originallyPublishedAt) | ||
| 232 | : null, | ||
| 233 | |||
| 234 | inputFileUpdatedAt: videoObject.uploadDate | ||
| 235 | ? new Date(videoObject.uploadDate) | ||
| 236 | : null, | ||
| 237 | |||
| 238 | updatedAt: new Date(videoObject.updated), | ||
| 239 | views: videoObject.views, | ||
| 240 | remote: true, | ||
| 241 | privacy | ||
| 242 | } | ||
| 243 | } | ||
| 244 | |||
| 245 | // --------------------------------------------------------------------------- | ||
| 246 | |||
| 247 | export { | ||
| 248 | getThumbnailFromIcons, | ||
| 249 | getPreviewFromIcons, | ||
| 250 | |||
| 251 | getTagsFromObject, | ||
| 252 | |||
| 253 | getFileAttributesFromUrl, | ||
| 254 | getStreamingPlaylistAttributesFromObject, | ||
| 255 | |||
| 256 | getLiveAttributesFromObject, | ||
| 257 | getCaptionAttributesFromObject, | ||
| 258 | getStoryboardAttributeFromObject, | ||
| 259 | |||
| 260 | getVideoAttributesFromObject | ||
| 261 | } | ||
| 262 | |||
| 263 | // --------------------------------------------------------------------------- | ||
| 264 | |||
| 265 | function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { | ||
| 266 | const urlMediaType = url.mediaType | ||
| 267 | |||
| 268 | return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/') | ||
| 269 | } | ||
| 270 | |||
| 271 | function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject { | ||
| 272 | return url && url.mediaType === 'application/x-mpegURL' | ||
| 273 | } | ||
| 274 | |||
| 275 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
| 276 | return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json' | ||
| 277 | } | ||
| 278 | |||
| 279 | function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { | ||
| 280 | return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' | ||
| 281 | } | ||
| 282 | |||
| 283 | function isAPHashTagObject (url: any): url is ActivityHashTagObject { | ||
| 284 | return url && url.type === 'Hashtag' | ||
| 285 | } | ||
diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts deleted file mode 100644 index 2418f45c2..000000000 --- a/server/lib/activitypub/videos/shared/trackers.ts +++ /dev/null | |||
| @@ -1,43 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize/types' | ||
| 2 | import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
| 3 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
| 4 | import { REMOTE_SCHEME } from '@server/initializers/constants' | ||
| 5 | import { TrackerModel } from '@server/models/server/tracker' | ||
| 6 | import { MVideo, MVideoWithHost } from '@server/types/models' | ||
| 7 | import { ActivityTrackerUrlObject, VideoObject } from '@shared/models' | ||
| 8 | import { buildRemoteVideoBaseUrl } from '../../url' | ||
| 9 | |||
| 10 | function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { | ||
| 11 | let wsFound = false | ||
| 12 | |||
| 13 | const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u)) | ||
| 14 | .map((u: ActivityTrackerUrlObject) => { | ||
| 15 | if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true | ||
| 16 | |||
| 17 | return u.href | ||
| 18 | }) | ||
| 19 | |||
| 20 | if (wsFound) return trackers | ||
| 21 | |||
| 22 | return [ | ||
| 23 | buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS), | ||
| 24 | buildRemoteVideoBaseUrl(video, '/tracker/announce') | ||
| 25 | ] | ||
| 26 | } | ||
| 27 | |||
| 28 | async function setVideoTrackers (options: { | ||
| 29 | video: MVideo | ||
| 30 | trackers: string[] | ||
| 31 | transaction: Transaction | ||
| 32 | }) { | ||
| 33 | const { video, trackers, transaction } = options | ||
| 34 | |||
| 35 | const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction) | ||
| 36 | |||
| 37 | await video.$set('Trackers', trackerInstances, { transaction }) | ||
| 38 | } | ||
| 39 | |||
| 40 | export { | ||
| 41 | getTrackerUrls, | ||
| 42 | setVideoTrackers | ||
| 43 | } | ||
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts deleted file mode 100644 index 7fe008419..000000000 --- a/server/lib/activitypub/videos/shared/url-to-object.ts +++ /dev/null | |||
| @@ -1,25 +0,0 @@ | |||
| 1 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | ||
| 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 3 | import { VideoObject } from '@shared/models' | ||
| 4 | import { fetchAP } from '../../activity' | ||
| 5 | import { checkUrlsSameHost } from '../../url' | ||
| 6 | |||
| 7 | const lTags = loggerTagsFactory('ap', 'video') | ||
| 8 | |||
| 9 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | ||
| 10 | logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl)) | ||
| 11 | |||
| 12 | const { statusCode, body } = await fetchAP<any>(videoUrl) | ||
| 13 | |||
| 14 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | ||
| 15 | logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) }) | ||
| 16 | |||
| 17 | return { statusCode, videoObject: undefined } | ||
| 18 | } | ||
| 19 | |||
| 20 | return { statusCode, videoObject: body } | ||
| 21 | } | ||
| 22 | |||
| 23 | export { | ||
| 24 | fetchRemoteVideo | ||
| 25 | } | ||
diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts deleted file mode 100644 index 7fb933559..000000000 --- a/server/lib/activitypub/videos/shared/video-sync-attributes.ts +++ /dev/null | |||
| @@ -1,107 +0,0 @@ | |||
| 1 | import { runInReadCommittedTransaction } from '@server/helpers/database-utils' | ||
| 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 3 | import { JobQueue } from '@server/lib/job-queue' | ||
| 4 | import { VideoModel } from '@server/models/video/video' | ||
| 5 | import { VideoCommentModel } from '@server/models/video/video-comment' | ||
| 6 | import { VideoShareModel } from '@server/models/video/video-share' | ||
| 7 | import { MVideo } from '@server/types/models' | ||
| 8 | import { ActivitypubHttpFetcherPayload, ActivityPubOrderedCollection, VideoObject } from '@shared/models' | ||
| 9 | import { fetchAP } from '../../activity' | ||
| 10 | import { crawlCollectionPage } from '../../crawl' | ||
| 11 | import { addVideoShares } from '../../share' | ||
| 12 | import { addVideoComments } from '../../video-comments' | ||
| 13 | |||
| 14 | const lTags = loggerTagsFactory('ap', 'video') | ||
| 15 | |||
| 16 | type SyncParam = { | ||
| 17 | rates: boolean | ||
| 18 | shares: boolean | ||
| 19 | comments: boolean | ||
| 20 | refreshVideo?: boolean | ||
| 21 | } | ||
| 22 | |||
| 23 | async function syncVideoExternalAttributes ( | ||
| 24 | video: MVideo, | ||
| 25 | fetchedVideo: VideoObject, | ||
| 26 | syncParam: Pick<SyncParam, 'rates' | 'shares' | 'comments'> | ||
| 27 | ) { | ||
| 28 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) | ||
| 29 | |||
| 30 | const ratePromise = updateVideoRates(video, fetchedVideo) | ||
| 31 | if (syncParam.rates) await ratePromise | ||
| 32 | |||
| 33 | await syncShares(video, fetchedVideo, syncParam.shares) | ||
| 34 | |||
| 35 | await syncComments(video, fetchedVideo, syncParam.comments) | ||
| 36 | } | ||
| 37 | |||
| 38 | async function updateVideoRates (video: MVideo, fetchedVideo: VideoObject) { | ||
| 39 | const [ likes, dislikes ] = await Promise.all([ | ||
| 40 | getRatesCount('like', video, fetchedVideo), | ||
| 41 | getRatesCount('dislike', video, fetchedVideo) | ||
| 42 | ]) | ||
| 43 | |||
| 44 | return runInReadCommittedTransaction(async t => { | ||
| 45 | await VideoModel.updateRatesOf(video.id, 'like', likes, t) | ||
| 46 | await VideoModel.updateRatesOf(video.id, 'dislike', dislikes, t) | ||
| 47 | }) | ||
| 48 | } | ||
| 49 | |||
| 50 | // --------------------------------------------------------------------------- | ||
| 51 | |||
| 52 | export { | ||
| 53 | SyncParam, | ||
| 54 | syncVideoExternalAttributes, | ||
| 55 | updateVideoRates | ||
| 56 | } | ||
| 57 | |||
| 58 | // --------------------------------------------------------------------------- | ||
| 59 | |||
| 60 | async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVideo: VideoObject) { | ||
| 61 | const uri = type === 'like' | ||
| 62 | ? fetchedVideo.likes | ||
| 63 | : fetchedVideo.dislikes | ||
| 64 | |||
| 65 | logger.info('Sync %s of video %s', type, video.url) | ||
| 66 | |||
| 67 | const { body } = await fetchAP<ActivityPubOrderedCollection<any>>(uri) | ||
| 68 | |||
| 69 | if (isNaN(body.totalItems)) { | ||
| 70 | logger.error('Cannot sync %s of video %s, totalItems is not a number', type, video.url, { body }) | ||
| 71 | return | ||
| 72 | } | ||
| 73 | |||
| 74 | return body.totalItems | ||
| 75 | } | ||
| 76 | |||
| 77 | function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { | ||
| 78 | const uri = fetchedVideo.shares | ||
| 79 | |||
| 80 | if (!isSync) { | ||
| 81 | return createJob({ uri, videoId: video.id, type: 'video-shares' }) | ||
| 82 | } | ||
| 83 | |||
| 84 | const handler = items => addVideoShares(items, video) | ||
| 85 | const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) | ||
| 86 | |||
| 87 | return crawlCollectionPage<string>(uri, handler, cleaner) | ||
| 88 | .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) | ||
| 89 | } | ||
| 90 | |||
| 91 | function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { | ||
| 92 | const uri = fetchedVideo.comments | ||
| 93 | |||
| 94 | if (!isSync) { | ||
| 95 | return createJob({ uri, videoId: video.id, type: 'video-comments' }) | ||
| 96 | } | ||
| 97 | |||
| 98 | const handler = items => addVideoComments(items) | ||
| 99 | const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) | ||
| 100 | |||
| 101 | return crawlCollectionPage<string>(uri, handler, cleaner) | ||
| 102 | .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) | ||
| 103 | } | ||
| 104 | |||
| 105 | function createJob (payload: ActivitypubHttpFetcherPayload) { | ||
| 106 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | ||
| 107 | } | ||
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts deleted file mode 100644 index acb087895..000000000 --- a/server/lib/activitypub/videos/updater.ts +++ /dev/null | |||
| @@ -1,180 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize/types' | ||
| 2 | import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' | ||
| 3 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
| 4 | import { Notifier } from '@server/lib/notifier' | ||
| 5 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
| 6 | import { Hooks } from '@server/lib/plugins/hooks' | ||
| 7 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | ||
| 8 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
| 9 | import { MActor, MChannelAccountLight, MChannelId, MVideoAccountLightBlacklistAllFiles, MVideoFullLight } from '@server/types/models' | ||
| 10 | import { VideoObject, VideoPrivacy } from '@shared/models' | ||
| 11 | import { APVideoAbstractBuilder, getVideoAttributesFromObject, updateVideoRates } from './shared' | ||
| 12 | |||
| 13 | export class APVideoUpdater extends APVideoAbstractBuilder { | ||
| 14 | private readonly wasPrivateVideo: boolean | ||
| 15 | private readonly wasUnlistedVideo: boolean | ||
| 16 | |||
| 17 | private readonly oldVideoChannel: MChannelAccountLight | ||
| 18 | |||
| 19 | protected lTags: LoggerTagsFn | ||
| 20 | |||
| 21 | constructor ( | ||
| 22 | protected readonly videoObject: VideoObject, | ||
| 23 | private readonly video: MVideoAccountLightBlacklistAllFiles | ||
| 24 | ) { | ||
| 25 | super() | ||
| 26 | |||
| 27 | this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE | ||
| 28 | this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED | ||
| 29 | |||
| 30 | this.oldVideoChannel = this.video.VideoChannel | ||
| 31 | |||
| 32 | this.lTags = loggerTagsFactory('ap', 'video', 'update', video.uuid, video.url) | ||
| 33 | } | ||
| 34 | |||
| 35 | async update (overrideTo?: string[]) { | ||
| 36 | logger.debug( | ||
| 37 | 'Updating remote video "%s".', this.videoObject.uuid, | ||
| 38 | { videoObject: this.videoObject, ...this.lTags() } | ||
| 39 | ) | ||
| 40 | |||
| 41 | const oldInputFileUpdatedAt = this.video.inputFileUpdatedAt | ||
| 42 | |||
| 43 | try { | ||
| 44 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() | ||
| 45 | |||
| 46 | const thumbnailModel = await this.setThumbnail(this.video) | ||
| 47 | |||
| 48 | this.checkChannelUpdateOrThrow(channelActor) | ||
| 49 | |||
| 50 | const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo) | ||
| 51 | |||
| 52 | if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) | ||
| 53 | |||
| 54 | await runInReadCommittedTransaction(async t => { | ||
| 55 | await this.setWebVideoFiles(videoUpdated, t) | ||
| 56 | await this.setStreamingPlaylists(videoUpdated, t) | ||
| 57 | }) | ||
| 58 | |||
| 59 | await Promise.all([ | ||
| 60 | runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), | ||
| 61 | runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), | ||
| 62 | runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)), | ||
| 63 | runInReadCommittedTransaction(t => { | ||
| 64 | return Promise.all([ | ||
| 65 | this.setPreview(videoUpdated, t), | ||
| 66 | this.setThumbnail(videoUpdated, t) | ||
| 67 | ]) | ||
| 68 | }), | ||
| 69 | this.setOrDeleteLive(videoUpdated) | ||
| 70 | ]) | ||
| 71 | |||
| 72 | await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) | ||
| 73 | |||
| 74 | await autoBlacklistVideoIfNeeded({ | ||
| 75 | video: videoUpdated, | ||
| 76 | user: undefined, | ||
| 77 | isRemote: true, | ||
| 78 | isNew: false, | ||
| 79 | isNewFile: oldInputFileUpdatedAt !== videoUpdated.inputFileUpdatedAt, | ||
| 80 | transaction: undefined | ||
| 81 | }) | ||
| 82 | |||
| 83 | await updateVideoRates(videoUpdated, this.videoObject) | ||
| 84 | |||
| 85 | // Notify our users? | ||
| 86 | if (this.wasPrivateVideo || this.wasUnlistedVideo) { | ||
| 87 | Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) | ||
| 88 | } | ||
| 89 | |||
| 90 | if (videoUpdated.isLive) { | ||
| 91 | PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated) | ||
| 92 | } | ||
| 93 | |||
| 94 | Hooks.runAction('action:activity-pub.remote-video.updated', { video: videoUpdated, videoAPObject: this.videoObject }) | ||
| 95 | |||
| 96 | logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags()) | ||
| 97 | |||
| 98 | return videoUpdated | ||
| 99 | } catch (err) { | ||
| 100 | await this.catchUpdateError(err) | ||
| 101 | } | ||
| 102 | } | ||
| 103 | |||
| 104 | // Check we can update the channel: we trust the remote server | ||
| 105 | private checkChannelUpdateOrThrow (newChannelActor: MActor) { | ||
| 106 | if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) { | ||
| 107 | throw new Error('Cannot check old channel/new channel validity because `serverId` is null') | ||
| 108 | } | ||
| 109 | |||
| 110 | if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) { | ||
| 111 | throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`) | ||
| 112 | } | ||
| 113 | } | ||
| 114 | |||
| 115 | private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) { | ||
| 116 | const to = overrideTo || this.videoObject.to | ||
| 117 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, to) | ||
| 118 | this.video.name = videoData.name | ||
| 119 | this.video.uuid = videoData.uuid | ||
| 120 | this.video.url = videoData.url | ||
| 121 | this.video.category = videoData.category | ||
| 122 | this.video.licence = videoData.licence | ||
| 123 | this.video.language = videoData.language | ||
| 124 | this.video.description = videoData.description | ||
| 125 | this.video.support = videoData.support | ||
| 126 | this.video.nsfw = videoData.nsfw | ||
| 127 | this.video.commentsEnabled = videoData.commentsEnabled | ||
| 128 | this.video.downloadEnabled = videoData.downloadEnabled | ||
| 129 | this.video.waitTranscoding = videoData.waitTranscoding | ||
| 130 | this.video.state = videoData.state | ||
| 131 | this.video.duration = videoData.duration | ||
| 132 | this.video.createdAt = videoData.createdAt | ||
| 133 | this.video.publishedAt = videoData.publishedAt | ||
| 134 | this.video.originallyPublishedAt = videoData.originallyPublishedAt | ||
| 135 | this.video.inputFileUpdatedAt = videoData.inputFileUpdatedAt | ||
| 136 | this.video.privacy = videoData.privacy | ||
| 137 | this.video.channelId = videoData.channelId | ||
| 138 | this.video.views = videoData.views | ||
| 139 | this.video.isLive = videoData.isLive | ||
| 140 | |||
| 141 | // Ensures we update the updatedAt attribute, even if main attributes did not change | ||
| 142 | this.video.changed('updatedAt', true) | ||
| 143 | |||
| 144 | return this.video.save({ transaction }) as Promise<MVideoFullLight> | ||
| 145 | } | ||
| 146 | |||
| 147 | private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) { | ||
| 148 | await this.insertOrReplaceCaptions(videoUpdated, t) | ||
| 149 | } | ||
| 150 | |||
| 151 | private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) { | ||
| 152 | await this.insertOrReplaceStoryboard(videoUpdated, t) | ||
| 153 | } | ||
| 154 | |||
| 155 | private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { | ||
| 156 | if (!this.video.isLive) return | ||
| 157 | |||
| 158 | if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction) | ||
| 159 | |||
| 160 | // Delete existing live if it exists | ||
| 161 | await VideoLiveModel.destroy({ | ||
| 162 | where: { | ||
| 163 | videoId: this.video.id | ||
| 164 | }, | ||
| 165 | transaction | ||
| 166 | }) | ||
| 167 | |||
| 168 | videoUpdated.VideoLive = null | ||
| 169 | } | ||
| 170 | |||
| 171 | private async catchUpdateError (err: Error) { | ||
| 172 | if (this.video !== undefined) { | ||
| 173 | await resetSequelizeInstance(this.video) | ||
| 174 | } | ||
| 175 | |||
| 176 | // This is just a debug because we will retry the insert | ||
| 177 | logger.debug('Cannot update the remote video.', { err, ...this.lTags() }) | ||
| 178 | throw err | ||
| 179 | } | ||
| 180 | } | ||
diff --git a/server/lib/actor-follow-health-cache.ts b/server/lib/actor-follow-health-cache.ts deleted file mode 100644 index 34357a97a..000000000 --- a/server/lib/actor-follow-health-cache.ts +++ /dev/null | |||
| @@ -1,86 +0,0 @@ | |||
| 1 | import { ACTOR_FOLLOW_SCORE } from '../initializers/constants' | ||
| 2 | import { logger } from '../helpers/logger' | ||
| 3 | |||
| 4 | // Cache follows scores, instead of writing them too often in database | ||
| 5 | // Keep data in memory, we don't really need Redis here as we don't really care to loose some scores | ||
| 6 | class ActorFollowHealthCache { | ||
| 7 | |||
| 8 | private static instance: ActorFollowHealthCache | ||
| 9 | |||
| 10 | private pendingFollowsScore: { [ url: string ]: number } = {} | ||
| 11 | |||
| 12 | private pendingBadServer = new Set<number>() | ||
| 13 | private pendingGoodServer = new Set<number>() | ||
| 14 | |||
| 15 | private readonly badInboxes = new Set<string>() | ||
| 16 | |||
| 17 | private constructor () {} | ||
| 18 | |||
| 19 | static get Instance () { | ||
| 20 | return this.instance || (this.instance = new this()) | ||
| 21 | } | ||
| 22 | |||
| 23 | updateActorFollowsHealth (goodInboxes: string[], badInboxes: string[]) { | ||
| 24 | this.badInboxes.clear() | ||
| 25 | |||
| 26 | if (goodInboxes.length === 0 && badInboxes.length === 0) return | ||
| 27 | |||
| 28 | logger.info( | ||
| 29 | 'Updating %d good actor follows and %d bad actor follows scores in cache.', | ||
| 30 | goodInboxes.length, badInboxes.length, { badInboxes } | ||
| 31 | ) | ||
| 32 | |||
| 33 | for (const goodInbox of goodInboxes) { | ||
| 34 | if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0 | ||
| 35 | |||
| 36 | this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS | ||
| 37 | } | ||
| 38 | |||
| 39 | for (const badInbox of badInboxes) { | ||
| 40 | if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0 | ||
| 41 | |||
| 42 | this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY | ||
| 43 | this.badInboxes.add(badInbox) | ||
| 44 | } | ||
| 45 | } | ||
| 46 | |||
| 47 | isBadInbox (inboxUrl: string) { | ||
| 48 | return this.badInboxes.has(inboxUrl) | ||
| 49 | } | ||
| 50 | |||
| 51 | addBadServerId (serverId: number) { | ||
| 52 | this.pendingBadServer.add(serverId) | ||
| 53 | } | ||
| 54 | |||
| 55 | getBadFollowingServerIds () { | ||
| 56 | return Array.from(this.pendingBadServer) | ||
| 57 | } | ||
| 58 | |||
| 59 | clearBadFollowingServerIds () { | ||
| 60 | this.pendingBadServer = new Set<number>() | ||
| 61 | } | ||
| 62 | |||
| 63 | addGoodServerId (serverId: number) { | ||
| 64 | this.pendingGoodServer.add(serverId) | ||
| 65 | } | ||
| 66 | |||
| 67 | getGoodFollowingServerIds () { | ||
| 68 | return Array.from(this.pendingGoodServer) | ||
| 69 | } | ||
| 70 | |||
| 71 | clearGoodFollowingServerIds () { | ||
| 72 | this.pendingGoodServer = new Set<number>() | ||
| 73 | } | ||
| 74 | |||
| 75 | getPendingFollowsScore () { | ||
| 76 | return this.pendingFollowsScore | ||
| 77 | } | ||
| 78 | |||
| 79 | clearPendingFollowsScore () { | ||
| 80 | this.pendingFollowsScore = {} | ||
| 81 | } | ||
| 82 | } | ||
| 83 | |||
| 84 | export { | ||
| 85 | ActorFollowHealthCache | ||
| 86 | } | ||
diff --git a/server/lib/actor-image.ts b/server/lib/actor-image.ts deleted file mode 100644 index e9bd148f6..000000000 --- a/server/lib/actor-image.ts +++ /dev/null | |||
| @@ -1,14 +0,0 @@ | |||
| 1 | import maxBy from 'lodash/maxBy' | ||
| 2 | |||
| 3 | function getBiggestActorImage <T extends { width: number }> (images: T[]) { | ||
| 4 | const image = maxBy(images, 'width') | ||
| 5 | |||
| 6 | // If width is null, maxBy won't return a value | ||
| 7 | if (!image) return images[0] | ||
| 8 | |||
| 9 | return image | ||
| 10 | } | ||
| 11 | |||
| 12 | export { | ||
| 13 | getBiggestActorImage | ||
| 14 | } | ||
diff --git a/server/lib/auth/external-auth.ts b/server/lib/auth/external-auth.ts deleted file mode 100644 index bc5b74257..000000000 --- a/server/lib/auth/external-auth.ts +++ /dev/null | |||
| @@ -1,231 +0,0 @@ | |||
| 1 | |||
| 2 | import { | ||
| 3 | isUserAdminFlagsValid, | ||
| 4 | isUserDisplayNameValid, | ||
| 5 | isUserRoleValid, | ||
| 6 | isUserUsernameValid, | ||
| 7 | isUserVideoQuotaDailyValid, | ||
| 8 | isUserVideoQuotaValid | ||
| 9 | } from '@server/helpers/custom-validators/users' | ||
| 10 | import { logger } from '@server/helpers/logger' | ||
| 11 | import { generateRandomString } from '@server/helpers/utils' | ||
| 12 | import { PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME } from '@server/initializers/constants' | ||
| 13 | import { PluginManager } from '@server/lib/plugins/plugin-manager' | ||
| 14 | import { OAuthTokenModel } from '@server/models/oauth/oauth-token' | ||
| 15 | import { MUser } from '@server/types/models' | ||
| 16 | import { | ||
| 17 | RegisterServerAuthenticatedResult, | ||
| 18 | RegisterServerAuthPassOptions, | ||
| 19 | RegisterServerExternalAuthenticatedResult | ||
| 20 | } from '@server/types/plugins/register-server-auth.model' | ||
| 21 | import { UserAdminFlag, UserRole } from '@shared/models' | ||
| 22 | import { BypassLogin } from './oauth-model' | ||
| 23 | |||
| 24 | export type ExternalUser = | ||
| 25 | Pick<MUser, 'username' | 'email' | 'role' | 'adminFlags' | 'videoQuotaDaily' | 'videoQuota'> & | ||
| 26 | { displayName: string } | ||
| 27 | |||
| 28 | // Token is the key, expiration date is the value | ||
| 29 | const authBypassTokens = new Map<string, { | ||
| 30 | expires: Date | ||
| 31 | user: ExternalUser | ||
| 32 | userUpdater: RegisterServerAuthenticatedResult['userUpdater'] | ||
| 33 | authName: string | ||
| 34 | npmName: string | ||
| 35 | }>() | ||
| 36 | |||
| 37 | async function onExternalUserAuthenticated (options: { | ||
| 38 | npmName: string | ||
| 39 | authName: string | ||
| 40 | authResult: RegisterServerExternalAuthenticatedResult | ||
| 41 | }) { | ||
| 42 | const { npmName, authName, authResult } = options | ||
| 43 | |||
| 44 | if (!authResult.req || !authResult.res) { | ||
| 45 | logger.error('Cannot authenticate external user for auth %s of plugin %s: no req or res are provided.', authName, npmName) | ||
| 46 | return | ||
| 47 | } | ||
| 48 | |||
| 49 | const { res } = authResult | ||
| 50 | |||
| 51 | if (!isAuthResultValid(npmName, authName, authResult)) { | ||
| 52 | res.redirect('/login?externalAuthError=true') | ||
| 53 | return | ||
| 54 | } | ||
| 55 | |||
| 56 | logger.info('Generating auth bypass token for %s in auth %s of plugin %s.', authResult.username, authName, npmName) | ||
| 57 | |||
| 58 | const bypassToken = await generateRandomString(32) | ||
| 59 | |||
| 60 | const expires = new Date() | ||
| 61 | expires.setTime(expires.getTime() + PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME) | ||
| 62 | |||
| 63 | const user = buildUserResult(authResult) | ||
| 64 | authBypassTokens.set(bypassToken, { | ||
| 65 | expires, | ||
| 66 | user, | ||
| 67 | npmName, | ||
| 68 | authName, | ||
| 69 | userUpdater: authResult.userUpdater | ||
| 70 | }) | ||
| 71 | |||
| 72 | // Cleanup expired tokens | ||
| 73 | const now = new Date() | ||
| 74 | for (const [ key, value ] of authBypassTokens) { | ||
| 75 | if (value.expires.getTime() < now.getTime()) { | ||
| 76 | authBypassTokens.delete(key) | ||
| 77 | } | ||
| 78 | } | ||
| 79 | |||
| 80 | res.redirect(`/login?externalAuthToken=${bypassToken}&username=${user.username}`) | ||
| 81 | } | ||
| 82 | |||
| 83 | async function getAuthNameFromRefreshGrant (refreshToken?: string) { | ||
| 84 | if (!refreshToken) return undefined | ||
| 85 | |||
| 86 | const tokenModel = await OAuthTokenModel.loadByRefreshToken(refreshToken) | ||
| 87 | |||
| 88 | return tokenModel?.authName | ||
| 89 | } | ||
| 90 | |||
| 91 | async function getBypassFromPasswordGrant (username: string, password: string): Promise<BypassLogin> { | ||
| 92 | const plugins = PluginManager.Instance.getIdAndPassAuths() | ||
| 93 | const pluginAuths: { npmName?: string, registerAuthOptions: RegisterServerAuthPassOptions }[] = [] | ||
| 94 | |||
| 95 | for (const plugin of plugins) { | ||
| 96 | const auths = plugin.idAndPassAuths | ||
| 97 | |||
| 98 | for (const auth of auths) { | ||
| 99 | pluginAuths.push({ | ||
| 100 | npmName: plugin.npmName, | ||
| 101 | registerAuthOptions: auth | ||
| 102 | }) | ||
| 103 | } | ||
| 104 | } | ||
| 105 | |||
| 106 | pluginAuths.sort((a, b) => { | ||
| 107 | const aWeight = a.registerAuthOptions.getWeight() | ||
| 108 | const bWeight = b.registerAuthOptions.getWeight() | ||
| 109 | |||
| 110 | // DESC weight order | ||
| 111 | if (aWeight === bWeight) return 0 | ||
| 112 | if (aWeight < bWeight) return 1 | ||
| 113 | return -1 | ||
| 114 | }) | ||
| 115 | |||
| 116 | const loginOptions = { | ||
| 117 | id: username, | ||
| 118 | password | ||
| 119 | } | ||
| 120 | |||
| 121 | for (const pluginAuth of pluginAuths) { | ||
| 122 | const authOptions = pluginAuth.registerAuthOptions | ||
| 123 | const authName = authOptions.authName | ||
| 124 | const npmName = pluginAuth.npmName | ||
| 125 | |||
| 126 | logger.debug( | ||
| 127 | 'Using auth method %s of plugin %s to login %s with weight %d.', | ||
| 128 | authName, npmName, loginOptions.id, authOptions.getWeight() | ||
| 129 | ) | ||
| 130 | |||
| 131 | try { | ||
| 132 | const loginResult = await authOptions.login(loginOptions) | ||
| 133 | |||
| 134 | if (!loginResult) continue | ||
| 135 | if (!isAuthResultValid(pluginAuth.npmName, authOptions.authName, loginResult)) continue | ||
| 136 | |||
| 137 | logger.info( | ||
| 138 | 'Login success with auth method %s of plugin %s for %s.', | ||
| 139 | authName, npmName, loginOptions.id | ||
| 140 | ) | ||
| 141 | |||
| 142 | return { | ||
| 143 | bypass: true, | ||
| 144 | pluginName: pluginAuth.npmName, | ||
| 145 | authName: authOptions.authName, | ||
| 146 | user: buildUserResult(loginResult), | ||
| 147 | userUpdater: loginResult.userUpdater | ||
| 148 | } | ||
| 149 | } catch (err) { | ||
| 150 | logger.error('Error in auth method %s of plugin %s', authOptions.authName, pluginAuth.npmName, { err }) | ||
| 151 | } | ||
| 152 | } | ||
| 153 | |||
| 154 | return undefined | ||
| 155 | } | ||
| 156 | |||
| 157 | function getBypassFromExternalAuth (username: string, externalAuthToken: string): BypassLogin { | ||
| 158 | const obj = authBypassTokens.get(externalAuthToken) | ||
| 159 | if (!obj) throw new Error('Cannot authenticate user with unknown bypass token') | ||
| 160 | |||
| 161 | const { expires, user, authName, npmName } = obj | ||
| 162 | |||
| 163 | const now = new Date() | ||
| 164 | if (now.getTime() > expires.getTime()) { | ||
| 165 | throw new Error('Cannot authenticate user with an expired external auth token') | ||
| 166 | } | ||
| 167 | |||
| 168 | if (user.username !== username) { | ||
| 169 | throw new Error(`Cannot authenticate user ${user.username} with invalid username ${username}`) | ||
| 170 | } | ||
| 171 | |||
| 172 | logger.info( | ||
| 173 | 'Auth success with external auth method %s of plugin %s for %s.', | ||
| 174 | authName, npmName, user.email | ||
| 175 | ) | ||
| 176 | |||
| 177 | return { | ||
| 178 | bypass: true, | ||
| 179 | pluginName: npmName, | ||
| 180 | authName, | ||
| 181 | userUpdater: obj.userUpdater, | ||
| 182 | user | ||
| 183 | } | ||
| 184 | } | ||
| 185 | |||
| 186 | function isAuthResultValid (npmName: string, authName: string, result: RegisterServerAuthenticatedResult) { | ||
| 187 | const returnError = (field: string) => { | ||
| 188 | logger.error('Auth method %s of plugin %s did not provide a valid %s.', authName, npmName, field, { [field]: result[field] }) | ||
| 189 | return false | ||
| 190 | } | ||
| 191 | |||
| 192 | if (!isUserUsernameValid(result.username)) return returnError('username') | ||
| 193 | if (!result.email) return returnError('email') | ||
| 194 | |||
| 195 | // Following fields are optional | ||
| 196 | if (result.role && !isUserRoleValid(result.role)) return returnError('role') | ||
| 197 | if (result.displayName && !isUserDisplayNameValid(result.displayName)) return returnError('displayName') | ||
| 198 | if (result.adminFlags && !isUserAdminFlagsValid(result.adminFlags)) return returnError('adminFlags') | ||
| 199 | if (result.videoQuota && !isUserVideoQuotaValid(result.videoQuota + '')) return returnError('videoQuota') | ||
| 200 | if (result.videoQuotaDaily && !isUserVideoQuotaDailyValid(result.videoQuotaDaily + '')) return returnError('videoQuotaDaily') | ||
| 201 | |||
| 202 | if (result.userUpdater && typeof result.userUpdater !== 'function') { | ||
| 203 | logger.error('Auth method %s of plugin %s did not provide a valid user updater function.', authName, npmName) | ||
| 204 | return false | ||
| 205 | } | ||
| 206 | |||
| 207 | return true | ||
| 208 | } | ||
| 209 | |||
| 210 | function buildUserResult (pluginResult: RegisterServerAuthenticatedResult) { | ||
| 211 | return { | ||
| 212 | username: pluginResult.username, | ||
| 213 | email: pluginResult.email, | ||
| 214 | role: pluginResult.role ?? UserRole.USER, | ||
| 215 | displayName: pluginResult.displayName || pluginResult.username, | ||
| 216 | |||
| 217 | adminFlags: pluginResult.adminFlags ?? UserAdminFlag.NONE, | ||
| 218 | |||
| 219 | videoQuota: pluginResult.videoQuota, | ||
| 220 | videoQuotaDaily: pluginResult.videoQuotaDaily | ||
| 221 | } | ||
| 222 | } | ||
| 223 | |||
| 224 | // --------------------------------------------------------------------------- | ||
| 225 | |||
| 226 | export { | ||
| 227 | onExternalUserAuthenticated, | ||
| 228 | getBypassFromExternalAuth, | ||
| 229 | getAuthNameFromRefreshGrant, | ||
| 230 | getBypassFromPasswordGrant | ||
| 231 | } | ||
diff --git a/server/lib/auth/oauth-model.ts b/server/lib/auth/oauth-model.ts deleted file mode 100644 index d3a5eccd5..000000000 --- a/server/lib/auth/oauth-model.ts +++ /dev/null | |||
| @@ -1,294 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { AccessDeniedError } from '@node-oauth/oauth2-server' | ||
| 3 | import { PluginManager } from '@server/lib/plugins/plugin-manager' | ||
| 4 | import { AccountModel } from '@server/models/account/account' | ||
| 5 | import { AuthenticatedResultUpdaterFieldName, RegisterServerAuthenticatedResult } from '@server/types' | ||
| 6 | import { MOAuthClient } from '@server/types/models' | ||
| 7 | import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' | ||
| 8 | import { MUser, MUserDefault } from '@server/types/models/user/user' | ||
| 9 | import { pick } from '@shared/core-utils' | ||
| 10 | import { AttributesOnly } from '@shared/typescript-utils' | ||
| 11 | import { logger } from '../../helpers/logger' | ||
| 12 | import { CONFIG } from '../../initializers/config' | ||
| 13 | import { OAuthClientModel } from '../../models/oauth/oauth-client' | ||
| 14 | import { OAuthTokenModel } from '../../models/oauth/oauth-token' | ||
| 15 | import { UserModel } from '../../models/user/user' | ||
| 16 | import { findAvailableLocalActorName } from '../local-actor' | ||
| 17 | import { buildUser, createUserAccountAndChannelAndPlaylist } from '../user' | ||
| 18 | import { ExternalUser } from './external-auth' | ||
| 19 | import { TokensCache } from './tokens-cache' | ||
| 20 | |||
| 21 | type TokenInfo = { | ||
| 22 | accessToken: string | ||
| 23 | refreshToken: string | ||
| 24 | accessTokenExpiresAt: Date | ||
| 25 | refreshTokenExpiresAt: Date | ||
| 26 | } | ||
| 27 | |||
| 28 | export type BypassLogin = { | ||
| 29 | bypass: boolean | ||
| 30 | pluginName: string | ||
| 31 | authName?: string | ||
| 32 | user: ExternalUser | ||
| 33 | userUpdater: RegisterServerAuthenticatedResult['userUpdater'] | ||
| 34 | } | ||
| 35 | |||
| 36 | async function getAccessToken (bearerToken: string) { | ||
| 37 | logger.debug('Getting access token.') | ||
| 38 | |||
| 39 | if (!bearerToken) return undefined | ||
| 40 | |||
| 41 | let tokenModel: MOAuthTokenUser | ||
| 42 | |||
| 43 | if (TokensCache.Instance.hasToken(bearerToken)) { | ||
| 44 | tokenModel = TokensCache.Instance.getByToken(bearerToken) | ||
| 45 | } else { | ||
| 46 | tokenModel = await OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) | ||
| 47 | |||
| 48 | if (tokenModel) TokensCache.Instance.setToken(tokenModel) | ||
| 49 | } | ||
| 50 | |||
| 51 | if (!tokenModel) return undefined | ||
| 52 | |||
| 53 | if (tokenModel.User.pluginAuth) { | ||
| 54 | const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'access') | ||
| 55 | |||
| 56 | if (valid !== true) return undefined | ||
| 57 | } | ||
| 58 | |||
| 59 | return tokenModel | ||
| 60 | } | ||
| 61 | |||
| 62 | function getClient (clientId: string, clientSecret: string) { | ||
| 63 | logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') | ||
| 64 | |||
| 65 | return OAuthClientModel.getByIdAndSecret(clientId, clientSecret) | ||
| 66 | } | ||
| 67 | |||
| 68 | async function getRefreshToken (refreshToken: string) { | ||
| 69 | logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') | ||
| 70 | |||
| 71 | const tokenInfo = await OAuthTokenModel.getByRefreshTokenAndPopulateClient(refreshToken) | ||
| 72 | if (!tokenInfo) return undefined | ||
| 73 | |||
| 74 | const tokenModel = tokenInfo.token | ||
| 75 | |||
| 76 | if (tokenModel.User.pluginAuth) { | ||
| 77 | const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'refresh') | ||
| 78 | |||
| 79 | if (valid !== true) return undefined | ||
| 80 | } | ||
| 81 | |||
| 82 | return tokenInfo | ||
| 83 | } | ||
| 84 | |||
| 85 | async function getUser (usernameOrEmail?: string, password?: string, bypassLogin?: BypassLogin) { | ||
| 86 | // Special treatment coming from a plugin | ||
| 87 | if (bypassLogin && bypassLogin.bypass === true) { | ||
| 88 | logger.info('Bypassing oauth login by plugin %s.', bypassLogin.pluginName) | ||
| 89 | |||
| 90 | let user = await UserModel.loadByEmail(bypassLogin.user.email) | ||
| 91 | |||
| 92 | if (!user) user = await createUserFromExternal(bypassLogin.pluginName, bypassLogin.user) | ||
| 93 | else user = await updateUserFromExternal(user, bypassLogin.user, bypassLogin.userUpdater) | ||
| 94 | |||
| 95 | // Cannot create a user | ||
| 96 | if (!user) throw new AccessDeniedError('Cannot create such user: an actor with that name already exists.') | ||
| 97 | |||
| 98 | // If the user does not belongs to a plugin, it was created before its installation | ||
| 99 | // Then we just go through a regular login process | ||
| 100 | if (user.pluginAuth !== null) { | ||
| 101 | // This user does not belong to this plugin, skip it | ||
| 102 | if (user.pluginAuth !== bypassLogin.pluginName) { | ||
| 103 | logger.info( | ||
| 104 | 'Cannot bypass oauth login by plugin %s because %s has another plugin auth method (%s).', | ||
| 105 | bypassLogin.pluginName, bypassLogin.user.email, user.pluginAuth | ||
| 106 | ) | ||
| 107 | |||
| 108 | return null | ||
| 109 | } | ||
| 110 | |||
| 111 | checkUserValidityOrThrow(user) | ||
| 112 | |||
| 113 | return user | ||
| 114 | } | ||
| 115 | } | ||
| 116 | |||
| 117 | logger.debug('Getting User (username/email: ' + usernameOrEmail + ', password: ******).') | ||
| 118 | |||
| 119 | const user = await UserModel.loadByUsernameOrEmail(usernameOrEmail) | ||
| 120 | |||
| 121 | // If we don't find the user, or if the user belongs to a plugin | ||
| 122 | if (!user || user.pluginAuth !== null || !password) return null | ||
| 123 | |||
| 124 | const passwordMatch = await user.isPasswordMatch(password) | ||
| 125 | if (passwordMatch !== true) return null | ||
| 126 | |||
| 127 | checkUserValidityOrThrow(user) | ||
| 128 | |||
| 129 | if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION && user.emailVerified === false) { | ||
| 130 | throw new AccessDeniedError('User email is not verified.') | ||
| 131 | } | ||
| 132 | |||
| 133 | return user | ||
| 134 | } | ||
| 135 | |||
| 136 | async function revokeToken ( | ||
| 137 | tokenInfo: { refreshToken: string }, | ||
| 138 | options: { | ||
| 139 | req?: express.Request | ||
| 140 | explicitLogout?: boolean | ||
| 141 | } = {} | ||
| 142 | ): Promise<{ success: boolean, redirectUrl?: string }> { | ||
| 143 | const { req, explicitLogout } = options | ||
| 144 | |||
| 145 | const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) | ||
| 146 | |||
| 147 | if (token) { | ||
| 148 | let redirectUrl: string | ||
| 149 | |||
| 150 | if (explicitLogout === true && token.User.pluginAuth && token.authName) { | ||
| 151 | redirectUrl = await PluginManager.Instance.onLogout(token.User.pluginAuth, token.authName, token.User, req) | ||
| 152 | } | ||
| 153 | |||
| 154 | TokensCache.Instance.clearCacheByToken(token.accessToken) | ||
| 155 | |||
| 156 | token.destroy() | ||
| 157 | .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) | ||
| 158 | |||
| 159 | return { success: true, redirectUrl } | ||
| 160 | } | ||
| 161 | |||
| 162 | return { success: false } | ||
| 163 | } | ||
| 164 | |||
| 165 | async function saveToken ( | ||
| 166 | token: TokenInfo, | ||
| 167 | client: MOAuthClient, | ||
| 168 | user: MUser, | ||
| 169 | options: { | ||
| 170 | refreshTokenAuthName?: string | ||
| 171 | bypassLogin?: BypassLogin | ||
| 172 | } = {} | ||
| 173 | ) { | ||
| 174 | const { refreshTokenAuthName, bypassLogin } = options | ||
| 175 | let authName: string = null | ||
| 176 | |||
| 177 | if (bypassLogin?.bypass === true) { | ||
| 178 | authName = bypassLogin.authName | ||
| 179 | } else if (refreshTokenAuthName) { | ||
| 180 | authName = refreshTokenAuthName | ||
| 181 | } | ||
| 182 | |||
| 183 | logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') | ||
| 184 | |||
| 185 | const tokenToCreate = { | ||
| 186 | accessToken: token.accessToken, | ||
| 187 | accessTokenExpiresAt: token.accessTokenExpiresAt, | ||
| 188 | refreshToken: token.refreshToken, | ||
| 189 | refreshTokenExpiresAt: token.refreshTokenExpiresAt, | ||
| 190 | authName, | ||
| 191 | oAuthClientId: client.id, | ||
| 192 | userId: user.id | ||
| 193 | } | ||
| 194 | |||
| 195 | const tokenCreated = await OAuthTokenModel.create(tokenToCreate) | ||
| 196 | |||
| 197 | user.lastLoginDate = new Date() | ||
| 198 | await user.save() | ||
| 199 | |||
| 200 | return { | ||
| 201 | accessToken: tokenCreated.accessToken, | ||
| 202 | accessTokenExpiresAt: tokenCreated.accessTokenExpiresAt, | ||
| 203 | refreshToken: tokenCreated.refreshToken, | ||
| 204 | refreshTokenExpiresAt: tokenCreated.refreshTokenExpiresAt, | ||
| 205 | client, | ||
| 206 | user, | ||
| 207 | accessTokenExpiresIn: buildExpiresIn(tokenCreated.accessTokenExpiresAt), | ||
| 208 | refreshTokenExpiresIn: buildExpiresIn(tokenCreated.refreshTokenExpiresAt) | ||
| 209 | } | ||
| 210 | } | ||
| 211 | |||
| 212 | export { | ||
| 213 | getAccessToken, | ||
| 214 | getClient, | ||
| 215 | getRefreshToken, | ||
| 216 | getUser, | ||
| 217 | revokeToken, | ||
| 218 | saveToken | ||
| 219 | } | ||
| 220 | |||
| 221 | // --------------------------------------------------------------------------- | ||
| 222 | |||
| 223 | async function createUserFromExternal (pluginAuth: string, userOptions: ExternalUser) { | ||
| 224 | const username = await findAvailableLocalActorName(userOptions.username) | ||
| 225 | |||
| 226 | const userToCreate = buildUser({ | ||
| 227 | ...pick(userOptions, [ 'email', 'role', 'adminFlags', 'videoQuota', 'videoQuotaDaily' ]), | ||
| 228 | |||
| 229 | username, | ||
| 230 | emailVerified: null, | ||
| 231 | password: null, | ||
| 232 | pluginAuth | ||
| 233 | }) | ||
| 234 | |||
| 235 | const { user } = await createUserAccountAndChannelAndPlaylist({ | ||
| 236 | userToCreate, | ||
| 237 | userDisplayName: userOptions.displayName | ||
| 238 | }) | ||
| 239 | |||
| 240 | return user | ||
| 241 | } | ||
| 242 | |||
| 243 | async function updateUserFromExternal ( | ||
| 244 | user: MUserDefault, | ||
| 245 | userOptions: ExternalUser, | ||
| 246 | userUpdater: RegisterServerAuthenticatedResult['userUpdater'] | ||
| 247 | ) { | ||
| 248 | if (!userUpdater) return user | ||
| 249 | |||
| 250 | { | ||
| 251 | type UserAttributeKeys = keyof AttributesOnly<UserModel> | ||
| 252 | const mappingKeys: { [ id in UserAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { | ||
| 253 | role: 'role', | ||
| 254 | adminFlags: 'adminFlags', | ||
| 255 | videoQuota: 'videoQuota', | ||
| 256 | videoQuotaDaily: 'videoQuotaDaily' | ||
| 257 | } | ||
| 258 | |||
| 259 | for (const modelKey of Object.keys(mappingKeys)) { | ||
| 260 | const pluginOptionKey = mappingKeys[modelKey] | ||
| 261 | |||
| 262 | const newValue = userUpdater({ fieldName: pluginOptionKey, currentValue: user[modelKey], newValue: userOptions[pluginOptionKey] }) | ||
| 263 | user.set(modelKey, newValue) | ||
| 264 | } | ||
| 265 | } | ||
| 266 | |||
| 267 | { | ||
| 268 | type AccountAttributeKeys = keyof Partial<AttributesOnly<AccountModel>> | ||
| 269 | const mappingKeys: { [ id in AccountAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { | ||
| 270 | name: 'displayName' | ||
| 271 | } | ||
| 272 | |||
| 273 | for (const modelKey of Object.keys(mappingKeys)) { | ||
| 274 | const optionKey = mappingKeys[modelKey] | ||
| 275 | |||
| 276 | const newValue = userUpdater({ fieldName: optionKey, currentValue: user.Account[modelKey], newValue: userOptions[optionKey] }) | ||
| 277 | user.Account.set(modelKey, newValue) | ||
| 278 | } | ||
| 279 | } | ||
| 280 | |||
| 281 | logger.debug('Updated user %s with plugin userUpdated function.', user.email, { user, userOptions }) | ||
| 282 | |||
| 283 | user.Account = await user.Account.save() | ||
| 284 | |||
| 285 | return user.save() | ||
| 286 | } | ||
| 287 | |||
| 288 | function checkUserValidityOrThrow (user: MUser) { | ||
| 289 | if (user.blocked) throw new AccessDeniedError('User is blocked.') | ||
| 290 | } | ||
| 291 | |||
| 292 | function buildExpiresIn (expiresAt: Date) { | ||
| 293 | return Math.floor((expiresAt.getTime() - new Date().getTime()) / 1000) | ||
| 294 | } | ||
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts deleted file mode 100644 index 887c4f7c9..000000000 --- a/server/lib/auth/oauth.ts +++ /dev/null | |||
| @@ -1,223 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import OAuth2Server, { | ||
| 3 | InvalidClientError, | ||
| 4 | InvalidGrantError, | ||
| 5 | InvalidRequestError, | ||
| 6 | Request, | ||
| 7 | Response, | ||
| 8 | UnauthorizedClientError, | ||
| 9 | UnsupportedGrantTypeError | ||
| 10 | } from '@node-oauth/oauth2-server' | ||
| 11 | import { randomBytesPromise } from '@server/helpers/core-utils' | ||
| 12 | import { isOTPValid } from '@server/helpers/otp' | ||
| 13 | import { CONFIG } from '@server/initializers/config' | ||
| 14 | import { UserRegistrationModel } from '@server/models/user/user-registration' | ||
| 15 | import { MOAuthClient } from '@server/types/models' | ||
| 16 | import { sha1 } from '@shared/extra-utils' | ||
| 17 | import { HttpStatusCode, ServerErrorCode, UserRegistrationState } from '@shared/models' | ||
| 18 | import { OTP } from '../../initializers/constants' | ||
| 19 | import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' | ||
| 20 | |||
| 21 | class MissingTwoFactorError extends Error { | ||
| 22 | code = HttpStatusCode.UNAUTHORIZED_401 | ||
| 23 | name = ServerErrorCode.MISSING_TWO_FACTOR | ||
| 24 | } | ||
| 25 | |||
| 26 | class InvalidTwoFactorError extends Error { | ||
| 27 | code = HttpStatusCode.BAD_REQUEST_400 | ||
| 28 | name = ServerErrorCode.INVALID_TWO_FACTOR | ||
| 29 | } | ||
| 30 | |||
| 31 | class RegistrationWaitingForApproval extends Error { | ||
| 32 | code = HttpStatusCode.BAD_REQUEST_400 | ||
| 33 | name = ServerErrorCode.ACCOUNT_WAITING_FOR_APPROVAL | ||
| 34 | } | ||
| 35 | |||
| 36 | class RegistrationApprovalRejected extends Error { | ||
| 37 | code = HttpStatusCode.BAD_REQUEST_400 | ||
| 38 | name = ServerErrorCode.ACCOUNT_APPROVAL_REJECTED | ||
| 39 | } | ||
| 40 | |||
| 41 | /** | ||
| 42 | * | ||
| 43 | * Reimplement some functions of OAuth2Server to inject external auth methods | ||
| 44 | * | ||
| 45 | */ | ||
| 46 | const oAuthServer = new OAuth2Server({ | ||
| 47 | // Wants seconds | ||
| 48 | accessTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN / 1000, | ||
| 49 | refreshTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN / 1000, | ||
| 50 | |||
| 51 | // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications | ||
| 52 | model: require('./oauth-model') | ||
| 53 | }) | ||
| 54 | |||
| 55 | // --------------------------------------------------------------------------- | ||
| 56 | |||
| 57 | async function handleOAuthToken (req: express.Request, options: { refreshTokenAuthName?: string, bypassLogin?: BypassLogin }) { | ||
| 58 | const request = new Request(req) | ||
| 59 | const { refreshTokenAuthName, bypassLogin } = options | ||
| 60 | |||
| 61 | if (request.method !== 'POST') { | ||
| 62 | throw new InvalidRequestError('Invalid request: method must be POST') | ||
| 63 | } | ||
| 64 | |||
| 65 | if (!request.is([ 'application/x-www-form-urlencoded' ])) { | ||
| 66 | throw new InvalidRequestError('Invalid request: content must be application/x-www-form-urlencoded') | ||
| 67 | } | ||
| 68 | |||
| 69 | const clientId = request.body.client_id | ||
| 70 | const clientSecret = request.body.client_secret | ||
| 71 | |||
| 72 | if (!clientId || !clientSecret) { | ||
| 73 | throw new InvalidClientError('Invalid client: cannot retrieve client credentials') | ||
| 74 | } | ||
| 75 | |||
| 76 | const client = await getClient(clientId, clientSecret) | ||
| 77 | if (!client) { | ||
| 78 | throw new InvalidClientError('Invalid client: client is invalid') | ||
| 79 | } | ||
| 80 | |||
| 81 | const grantType = request.body.grant_type | ||
| 82 | if (!grantType) { | ||
| 83 | throw new InvalidRequestError('Missing parameter: `grant_type`') | ||
| 84 | } | ||
| 85 | |||
| 86 | if (![ 'password', 'refresh_token' ].includes(grantType)) { | ||
| 87 | throw new UnsupportedGrantTypeError('Unsupported grant type: `grant_type` is invalid') | ||
| 88 | } | ||
| 89 | |||
| 90 | if (!client.grants.includes(grantType)) { | ||
| 91 | throw new UnauthorizedClientError('Unauthorized client: `grant_type` is invalid') | ||
| 92 | } | ||
| 93 | |||
| 94 | if (grantType === 'password') { | ||
| 95 | return handlePasswordGrant({ | ||
| 96 | request, | ||
| 97 | client, | ||
| 98 | bypassLogin | ||
| 99 | }) | ||
| 100 | } | ||
| 101 | |||
| 102 | return handleRefreshGrant({ | ||
| 103 | request, | ||
| 104 | client, | ||
| 105 | refreshTokenAuthName | ||
| 106 | }) | ||
| 107 | } | ||
| 108 | |||
| 109 | function handleOAuthAuthenticate ( | ||
| 110 | req: express.Request, | ||
| 111 | res: express.Response | ||
| 112 | ) { | ||
| 113 | return oAuthServer.authenticate(new Request(req), new Response(res)) | ||
| 114 | } | ||
| 115 | |||
| 116 | export { | ||
| 117 | MissingTwoFactorError, | ||
| 118 | InvalidTwoFactorError, | ||
| 119 | |||
| 120 | handleOAuthToken, | ||
| 121 | handleOAuthAuthenticate | ||
| 122 | } | ||
| 123 | |||
| 124 | // --------------------------------------------------------------------------- | ||
| 125 | |||
| 126 | async function handlePasswordGrant (options: { | ||
| 127 | request: Request | ||
| 128 | client: MOAuthClient | ||
| 129 | bypassLogin?: BypassLogin | ||
| 130 | }) { | ||
| 131 | const { request, client, bypassLogin } = options | ||
| 132 | |||
| 133 | if (!request.body.username) { | ||
| 134 | throw new InvalidRequestError('Missing parameter: `username`') | ||
| 135 | } | ||
| 136 | |||
| 137 | if (!bypassLogin && !request.body.password) { | ||
| 138 | throw new InvalidRequestError('Missing parameter: `password`') | ||
| 139 | } | ||
| 140 | |||
| 141 | const user = await getUser(request.body.username, request.body.password, bypassLogin) | ||
| 142 | if (!user) { | ||
| 143 | const registration = await UserRegistrationModel.loadByEmailOrUsername(request.body.username) | ||
| 144 | |||
| 145 | if (registration?.state === UserRegistrationState.REJECTED) { | ||
| 146 | throw new RegistrationApprovalRejected('Registration approval for this account has been rejected') | ||
| 147 | } else if (registration?.state === UserRegistrationState.PENDING) { | ||
| 148 | throw new RegistrationWaitingForApproval('Registration for this account is awaiting approval') | ||
| 149 | } | ||
| 150 | |||
| 151 | throw new InvalidGrantError('Invalid grant: user credentials are invalid') | ||
| 152 | } | ||
| 153 | |||
| 154 | if (user.otpSecret) { | ||
| 155 | if (!request.headers[OTP.HEADER_NAME]) { | ||
| 156 | throw new MissingTwoFactorError('Missing two factor header') | ||
| 157 | } | ||
| 158 | |||
| 159 | if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) { | ||
| 160 | throw new InvalidTwoFactorError('Invalid two factor header') | ||
| 161 | } | ||
| 162 | } | ||
| 163 | |||
| 164 | const token = await buildToken() | ||
| 165 | |||
| 166 | return saveToken(token, client, user, { bypassLogin }) | ||
| 167 | } | ||
| 168 | |||
| 169 | async function handleRefreshGrant (options: { | ||
| 170 | request: Request | ||
| 171 | client: MOAuthClient | ||
| 172 | refreshTokenAuthName: string | ||
| 173 | }) { | ||
| 174 | const { request, client, refreshTokenAuthName } = options | ||
| 175 | |||
| 176 | if (!request.body.refresh_token) { | ||
| 177 | throw new InvalidRequestError('Missing parameter: `refresh_token`') | ||
| 178 | } | ||
| 179 | |||
| 180 | const refreshToken = await getRefreshToken(request.body.refresh_token) | ||
| 181 | |||
| 182 | if (!refreshToken) { | ||
| 183 | throw new InvalidGrantError('Invalid grant: refresh token is invalid') | ||
| 184 | } | ||
| 185 | |||
| 186 | if (refreshToken.client.id !== client.id) { | ||
| 187 | throw new InvalidGrantError('Invalid grant: refresh token is invalid') | ||
| 188 | } | ||
| 189 | |||
| 190 | if (refreshToken.refreshTokenExpiresAt && refreshToken.refreshTokenExpiresAt < new Date()) { | ||
| 191 | throw new InvalidGrantError('Invalid grant: refresh token has expired') | ||
| 192 | } | ||
| 193 | |||
| 194 | await revokeToken({ refreshToken: refreshToken.refreshToken }) | ||
| 195 | |||
| 196 | const token = await buildToken() | ||
| 197 | |||
| 198 | return saveToken(token, client, refreshToken.user, { refreshTokenAuthName }) | ||
| 199 | } | ||
| 200 | |||
| 201 | function generateRandomToken () { | ||
| 202 | return randomBytesPromise(256) | ||
| 203 | .then(buffer => sha1(buffer)) | ||
| 204 | } | ||
| 205 | |||
| 206 | function getTokenExpiresAt (type: 'access' | 'refresh') { | ||
| 207 | const lifetime = type === 'access' | ||
| 208 | ? CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN | ||
| 209 | : CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN | ||
| 210 | |||
| 211 | return new Date(Date.now() + lifetime) | ||
| 212 | } | ||
| 213 | |||
| 214 | async function buildToken () { | ||
| 215 | const [ accessToken, refreshToken ] = await Promise.all([ generateRandomToken(), generateRandomToken() ]) | ||
| 216 | |||
| 217 | return { | ||
| 218 | accessToken, | ||
| 219 | refreshToken, | ||
| 220 | accessTokenExpiresAt: getTokenExpiresAt('access'), | ||
| 221 | refreshTokenExpiresAt: getTokenExpiresAt('refresh') | ||
| 222 | } | ||
| 223 | } | ||
diff --git a/server/lib/auth/tokens-cache.ts b/server/lib/auth/tokens-cache.ts deleted file mode 100644 index e7b12159b..000000000 --- a/server/lib/auth/tokens-cache.ts +++ /dev/null | |||
| @@ -1,52 +0,0 @@ | |||
| 1 | import { LRUCache } from 'lru-cache' | ||
| 2 | import { MOAuthTokenUser } from '@server/types/models' | ||
| 3 | import { LRU_CACHE } from '../../initializers/constants' | ||
| 4 | |||
| 5 | export class TokensCache { | ||
| 6 | |||
| 7 | private static instance: TokensCache | ||
| 8 | |||
| 9 | private readonly accessTokenCache = new LRUCache<string, MOAuthTokenUser>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) | ||
| 10 | private readonly userHavingToken = new LRUCache<number, string>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) | ||
| 11 | |||
| 12 | private constructor () { } | ||
| 13 | |||
| 14 | static get Instance () { | ||
| 15 | return this.instance || (this.instance = new this()) | ||
| 16 | } | ||
| 17 | |||
| 18 | hasToken (token: string) { | ||
| 19 | return this.accessTokenCache.has(token) | ||
| 20 | } | ||
| 21 | |||
| 22 | getByToken (token: string) { | ||
| 23 | return this.accessTokenCache.get(token) | ||
| 24 | } | ||
| 25 | |||
| 26 | setToken (token: MOAuthTokenUser) { | ||
| 27 | this.accessTokenCache.set(token.accessToken, token) | ||
| 28 | this.userHavingToken.set(token.userId, token.accessToken) | ||
| 29 | } | ||
| 30 | |||
| 31 | deleteUserToken (userId: number) { | ||
| 32 | this.clearCacheByUserId(userId) | ||
| 33 | } | ||
| 34 | |||
| 35 | clearCacheByUserId (userId: number) { | ||
| 36 | const token = this.userHavingToken.get(userId) | ||
| 37 | |||
| 38 | if (token !== undefined) { | ||
| 39 | this.accessTokenCache.delete(token) | ||
| 40 | this.userHavingToken.delete(userId) | ||
| 41 | } | ||
| 42 | } | ||
| 43 | |||
| 44 | clearCacheByToken (token: string) { | ||
| 45 | const tokenModel = this.accessTokenCache.get(token) | ||
| 46 | |||
| 47 | if (tokenModel !== undefined) { | ||
| 48 | this.userHavingToken.delete(tokenModel.userId) | ||
| 49 | this.accessTokenCache.delete(token) | ||
| 50 | } | ||
| 51 | } | ||
| 52 | } | ||
diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts deleted file mode 100644 index 009e229ce..000000000 --- a/server/lib/blocklist.ts +++ /dev/null | |||
| @@ -1,62 +0,0 @@ | |||
| 1 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 2 | import { getServerActor } from '@server/models/application/application' | ||
| 3 | import { MAccountBlocklist, MAccountId, MAccountHost, MServerBlocklist } from '@server/types/models' | ||
| 4 | import { AccountBlocklistModel } from '../models/account/account-blocklist' | ||
| 5 | import { ServerBlocklistModel } from '../models/server/server-blocklist' | ||
| 6 | |||
| 7 | function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { | ||
| 8 | return sequelizeTypescript.transaction(async t => { | ||
| 9 | return AccountBlocklistModel.upsert({ | ||
| 10 | accountId: byAccountId, | ||
| 11 | targetAccountId | ||
| 12 | }, { transaction: t }) | ||
| 13 | }) | ||
| 14 | } | ||
| 15 | |||
| 16 | function addServerInBlocklist (byAccountId: number, targetServerId: number) { | ||
| 17 | return sequelizeTypescript.transaction(async t => { | ||
| 18 | return ServerBlocklistModel.upsert({ | ||
| 19 | accountId: byAccountId, | ||
| 20 | targetServerId | ||
| 21 | }, { transaction: t }) | ||
| 22 | }) | ||
| 23 | } | ||
| 24 | |||
| 25 | function removeAccountFromBlocklist (accountBlock: MAccountBlocklist) { | ||
| 26 | return sequelizeTypescript.transaction(async t => { | ||
| 27 | return accountBlock.destroy({ transaction: t }) | ||
| 28 | }) | ||
| 29 | } | ||
| 30 | |||
| 31 | function removeServerFromBlocklist (serverBlock: MServerBlocklist) { | ||
| 32 | return sequelizeTypescript.transaction(async t => { | ||
| 33 | return serverBlock.destroy({ transaction: t }) | ||
| 34 | }) | ||
| 35 | } | ||
| 36 | |||
| 37 | async function isBlockedByServerOrAccount (targetAccount: MAccountHost, userAccount?: MAccountId) { | ||
| 38 | const serverAccountId = (await getServerActor()).Account.id | ||
| 39 | const sourceAccounts = [ serverAccountId ] | ||
| 40 | |||
| 41 | if (userAccount) sourceAccounts.push(userAccount.id) | ||
| 42 | |||
| 43 | const accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, targetAccount.id) | ||
| 44 | if (accountMutedHash[serverAccountId] || (userAccount && accountMutedHash[userAccount.id])) { | ||
| 45 | return true | ||
| 46 | } | ||
| 47 | |||
| 48 | const instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, targetAccount.Actor.serverId) | ||
| 49 | if (instanceMutedHash[serverAccountId] || (userAccount && instanceMutedHash[userAccount.id])) { | ||
| 50 | return true | ||
| 51 | } | ||
| 52 | |||
| 53 | return false | ||
| 54 | } | ||
| 55 | |||
| 56 | export { | ||
| 57 | addAccountInBlocklist, | ||
| 58 | addServerInBlocklist, | ||
| 59 | removeAccountFromBlocklist, | ||
| 60 | removeServerFromBlocklist, | ||
| 61 | isBlockedByServerOrAccount | ||
| 62 | } | ||
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts deleted file mode 100644 index 8e0c9e328..000000000 --- a/server/lib/client-html.ts +++ /dev/null | |||
| @@ -1,623 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { pathExists, readFile } from 'fs-extra' | ||
| 3 | import { truncate } from 'lodash' | ||
| 4 | import { join } from 'path' | ||
| 5 | import validator from 'validator' | ||
| 6 | import { isTestOrDevInstance } from '@server/helpers/core-utils' | ||
| 7 | import { toCompleteUUID } from '@server/helpers/custom-validators/misc' | ||
| 8 | import { mdToOneLinePlainText } from '@server/helpers/markdown' | ||
| 9 | import { ActorImageModel } from '@server/models/actor/actor-image' | ||
| 10 | import { root } from '@shared/core-utils' | ||
| 11 | import { escapeHTML } from '@shared/core-utils/renderer' | ||
| 12 | import { sha256 } from '@shared/extra-utils' | ||
| 13 | import { HTMLServerConfig } from '@shared/models' | ||
| 14 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' | ||
| 15 | import { HttpStatusCode } from '../../shared/models/http/http-error-codes' | ||
| 16 | import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' | ||
| 17 | import { logger } from '../helpers/logger' | ||
| 18 | import { CONFIG } from '../initializers/config' | ||
| 19 | import { | ||
| 20 | ACCEPT_HEADERS, | ||
| 21 | CUSTOM_HTML_TAG_COMMENTS, | ||
| 22 | EMBED_SIZE, | ||
| 23 | FILES_CONTENT_HASH, | ||
| 24 | PLUGIN_GLOBAL_CSS_PATH, | ||
| 25 | WEBSERVER | ||
| 26 | } from '../initializers/constants' | ||
| 27 | import { AccountModel } from '../models/account/account' | ||
| 28 | import { VideoModel } from '../models/video/video' | ||
| 29 | import { VideoChannelModel } from '../models/video/video-channel' | ||
| 30 | import { VideoPlaylistModel } from '../models/video/video-playlist' | ||
| 31 | import { MAccountHost, MChannelHost, MVideo, MVideoPlaylist } from '../types/models' | ||
| 32 | import { getActivityStreamDuration } from './activitypub/activity' | ||
| 33 | import { getBiggestActorImage } from './actor-image' | ||
| 34 | import { Hooks } from './plugins/hooks' | ||
| 35 | import { ServerConfigManager } from './server-config-manager' | ||
| 36 | import { isVideoInPrivateDirectory } from './video-privacy' | ||
| 37 | |||
| 38 | type Tags = { | ||
| 39 | ogType: string | ||
| 40 | twitterCard: 'player' | 'summary' | 'summary_large_image' | ||
| 41 | schemaType: string | ||
| 42 | |||
| 43 | list?: { | ||
| 44 | numberOfItems: number | ||
| 45 | } | ||
| 46 | |||
| 47 | escapedSiteName: string | ||
| 48 | escapedTitle: string | ||
| 49 | escapedTruncatedDescription: string | ||
| 50 | |||
| 51 | url: string | ||
| 52 | originUrl: string | ||
| 53 | |||
| 54 | disallowIndexation?: boolean | ||
| 55 | |||
| 56 | embed?: { | ||
| 57 | url: string | ||
| 58 | createdAt: string | ||
| 59 | duration?: string | ||
| 60 | views?: number | ||
| 61 | } | ||
| 62 | |||
| 63 | image: { | ||
| 64 | url: string | ||
| 65 | width?: number | ||
| 66 | height?: number | ||
| 67 | } | ||
| 68 | } | ||
| 69 | |||
| 70 | type HookContext = { | ||
| 71 | video?: MVideo | ||
| 72 | playlist?: MVideoPlaylist | ||
| 73 | } | ||
| 74 | |||
| 75 | class ClientHtml { | ||
| 76 | |||
| 77 | private static htmlCache: { [path: string]: string } = {} | ||
| 78 | |||
| 79 | static invalidCache () { | ||
| 80 | logger.info('Cleaning HTML cache.') | ||
| 81 | |||
| 82 | ClientHtml.htmlCache = {} | ||
| 83 | } | ||
| 84 | |||
| 85 | static async getDefaultHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { | ||
| 86 | const html = paramLang | ||
| 87 | ? await ClientHtml.getIndexHTML(req, res, paramLang) | ||
| 88 | : await ClientHtml.getIndexHTML(req, res) | ||
| 89 | |||
| 90 | let customHtml = ClientHtml.addTitleTag(html) | ||
| 91 | customHtml = ClientHtml.addDescriptionTag(customHtml) | ||
| 92 | |||
| 93 | return customHtml | ||
| 94 | } | ||
| 95 | |||
| 96 | static async getWatchHTMLPage (videoIdArg: string, req: express.Request, res: express.Response) { | ||
| 97 | const videoId = toCompleteUUID(videoIdArg) | ||
| 98 | |||
| 99 | // Let Angular application handle errors | ||
| 100 | if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) { | ||
| 101 | res.status(HttpStatusCode.NOT_FOUND_404) | ||
| 102 | return ClientHtml.getIndexHTML(req, res) | ||
| 103 | } | ||
| 104 | |||
| 105 | const [ html, video ] = await Promise.all([ | ||
| 106 | ClientHtml.getIndexHTML(req, res), | ||
| 107 | VideoModel.loadWithBlacklist(videoId) | ||
| 108 | ]) | ||
| 109 | |||
| 110 | // Let Angular application handle errors | ||
| 111 | if (!video || isVideoInPrivateDirectory(video.privacy) || video.VideoBlacklist) { | ||
| 112 | res.status(HttpStatusCode.NOT_FOUND_404) | ||
| 113 | return html | ||
| 114 | } | ||
| 115 | const escapedTruncatedDescription = buildEscapedTruncatedDescription(video.description) | ||
| 116 | |||
| 117 | let customHtml = ClientHtml.addTitleTag(html, video.name) | ||
| 118 | customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) | ||
| 119 | |||
| 120 | const url = WEBSERVER.URL + video.getWatchStaticPath() | ||
| 121 | const originUrl = video.url | ||
| 122 | const title = video.name | ||
| 123 | const siteName = CONFIG.INSTANCE.NAME | ||
| 124 | |||
| 125 | const image = { | ||
| 126 | url: WEBSERVER.URL + video.getPreviewStaticPath() | ||
| 127 | } | ||
| 128 | |||
| 129 | const embed = { | ||
| 130 | url: WEBSERVER.URL + video.getEmbedStaticPath(), | ||
| 131 | createdAt: video.createdAt.toISOString(), | ||
| 132 | duration: getActivityStreamDuration(video.duration), | ||
| 133 | views: video.views | ||
| 134 | } | ||
| 135 | |||
| 136 | const ogType = 'video' | ||
| 137 | const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary_large_image' | ||
| 138 | const schemaType = 'VideoObject' | ||
| 139 | |||
| 140 | customHtml = await ClientHtml.addTags(customHtml, { | ||
| 141 | url, | ||
| 142 | originUrl, | ||
| 143 | escapedSiteName: escapeHTML(siteName), | ||
| 144 | escapedTitle: escapeHTML(title), | ||
| 145 | escapedTruncatedDescription, | ||
| 146 | disallowIndexation: video.privacy !== VideoPrivacy.PUBLIC, | ||
| 147 | image, | ||
| 148 | embed, | ||
| 149 | ogType, | ||
| 150 | twitterCard, | ||
| 151 | schemaType | ||
| 152 | }, { video }) | ||
| 153 | |||
| 154 | return customHtml | ||
| 155 | } | ||
| 156 | |||
| 157 | static async getWatchPlaylistHTMLPage (videoPlaylistIdArg: string, req: express.Request, res: express.Response) { | ||
| 158 | const videoPlaylistId = toCompleteUUID(videoPlaylistIdArg) | ||
| 159 | |||
| 160 | // Let Angular application handle errors | ||
| 161 | if (!validator.isInt(videoPlaylistId) && !validator.isUUID(videoPlaylistId, 4)) { | ||
| 162 | res.status(HttpStatusCode.NOT_FOUND_404) | ||
| 163 | return ClientHtml.getIndexHTML(req, res) | ||
| 164 | } | ||
| 165 | |||
| 166 | const [ html, videoPlaylist ] = await Promise.all([ | ||
| 167 | ClientHtml.getIndexHTML(req, res), | ||
| 168 | VideoPlaylistModel.loadWithAccountAndChannel(videoPlaylistId, null) | ||
| 169 | ]) | ||
| 170 | |||
| 171 | // Let Angular application handle errors | ||
| 172 | if (!videoPlaylist || videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { | ||
| 173 | res.status(HttpStatusCode.NOT_FOUND_404) | ||
| 174 | return html | ||
| 175 | } | ||
| 176 | |||
| 177 | const escapedTruncatedDescription = buildEscapedTruncatedDescription(videoPlaylist.description) | ||
| 178 | |||
| 179 | let customHtml = ClientHtml.addTitleTag(html, videoPlaylist.name) | ||
| 180 | customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) | ||
| 181 | |||
| 182 | const url = WEBSERVER.URL + videoPlaylist.getWatchStaticPath() | ||
| 183 | const originUrl = videoPlaylist.url | ||
| 184 | const title = videoPlaylist.name | ||
| 185 | const siteName = CONFIG.INSTANCE.NAME | ||
| 186 | |||
| 187 | const image = { | ||
| 188 | url: videoPlaylist.getThumbnailUrl() | ||
| 189 | } | ||
| 190 | |||
| 191 | const embed = { | ||
| 192 | url: WEBSERVER.URL + videoPlaylist.getEmbedStaticPath(), | ||
| 193 | createdAt: videoPlaylist.createdAt.toISOString() | ||
| 194 | } | ||
| 195 | |||
| 196 | const list = { | ||
| 197 | numberOfItems: videoPlaylist.get('videosLength') as number | ||
| 198 | } | ||
| 199 | |||
| 200 | const ogType = 'video' | ||
| 201 | const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary' | ||
| 202 | const schemaType = 'ItemList' | ||
| 203 | |||
| 204 | customHtml = await ClientHtml.addTags(customHtml, { | ||
| 205 | url, | ||
| 206 | originUrl, | ||
| 207 | escapedSiteName: escapeHTML(siteName), | ||
| 208 | escapedTitle: escapeHTML(title), | ||
| 209 | escapedTruncatedDescription, | ||
| 210 | disallowIndexation: videoPlaylist.privacy !== VideoPlaylistPrivacy.PUBLIC, | ||
| 211 | embed, | ||
| 212 | image, | ||
| 213 | list, | ||
| 214 | ogType, | ||
| 215 | twitterCard, | ||
| 216 | schemaType | ||
| 217 | }, { playlist: videoPlaylist }) | ||
| 218 | |||
| 219 | return customHtml | ||
| 220 | } | ||
| 221 | |||
| 222 | static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | ||
| 223 | const accountModelPromise = AccountModel.loadByNameWithHost(nameWithHost) | ||
| 224 | return this.getAccountOrChannelHTMLPage(() => accountModelPromise, req, res) | ||
| 225 | } | ||
| 226 | |||
| 227 | static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | ||
| 228 | const videoChannelModelPromise = VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) | ||
| 229 | return this.getAccountOrChannelHTMLPage(() => videoChannelModelPromise, req, res) | ||
| 230 | } | ||
| 231 | |||
| 232 | static async getActorHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | ||
| 233 | const [ account, channel ] = await Promise.all([ | ||
| 234 | AccountModel.loadByNameWithHost(nameWithHost), | ||
| 235 | VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) | ||
| 236 | ]) | ||
| 237 | |||
| 238 | return this.getAccountOrChannelHTMLPage(() => Promise.resolve(account || channel), req, res) | ||
| 239 | } | ||
| 240 | |||
| 241 | static async getEmbedHTML () { | ||
| 242 | const path = ClientHtml.getEmbedPath() | ||
| 243 | |||
| 244 | // Disable HTML cache in dev mode because webpack can regenerate JS files | ||
| 245 | if (!isTestOrDevInstance() && ClientHtml.htmlCache[path]) { | ||
| 246 | return ClientHtml.htmlCache[path] | ||
| 247 | } | ||
| 248 | |||
| 249 | const buffer = await readFile(path) | ||
| 250 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() | ||
| 251 | |||
| 252 | let html = buffer.toString() | ||
| 253 | html = await ClientHtml.addAsyncPluginCSS(html) | ||
| 254 | html = ClientHtml.addCustomCSS(html) | ||
| 255 | html = ClientHtml.addTitleTag(html) | ||
| 256 | html = ClientHtml.addDescriptionTag(html) | ||
| 257 | html = ClientHtml.addServerConfig(html, serverConfig) | ||
| 258 | |||
| 259 | ClientHtml.htmlCache[path] = html | ||
| 260 | |||
| 261 | return html | ||
| 262 | } | ||
| 263 | |||
| 264 | private static async getAccountOrChannelHTMLPage ( | ||
| 265 | loader: () => Promise<MAccountHost | MChannelHost>, | ||
| 266 | req: express.Request, | ||
| 267 | res: express.Response | ||
| 268 | ) { | ||
| 269 | const [ html, entity ] = await Promise.all([ | ||
| 270 | ClientHtml.getIndexHTML(req, res), | ||
| 271 | loader() | ||
| 272 | ]) | ||
| 273 | |||
| 274 | // Let Angular application handle errors | ||
| 275 | if (!entity) { | ||
| 276 | res.status(HttpStatusCode.NOT_FOUND_404) | ||
| 277 | return ClientHtml.getIndexHTML(req, res) | ||
| 278 | } | ||
| 279 | |||
| 280 | const escapedTruncatedDescription = buildEscapedTruncatedDescription(entity.description) | ||
| 281 | |||
| 282 | let customHtml = ClientHtml.addTitleTag(html, entity.getDisplayName()) | ||
| 283 | customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) | ||
| 284 | |||
| 285 | const url = entity.getClientUrl() | ||
| 286 | const originUrl = entity.Actor.url | ||
| 287 | const siteName = CONFIG.INSTANCE.NAME | ||
| 288 | const title = entity.getDisplayName() | ||
| 289 | |||
| 290 | const avatar = getBiggestActorImage(entity.Actor.Avatars) | ||
| 291 | const image = { | ||
| 292 | url: ActorImageModel.getImageUrl(avatar), | ||
| 293 | width: avatar?.width, | ||
| 294 | height: avatar?.height | ||
| 295 | } | ||
| 296 | |||
| 297 | const ogType = 'website' | ||
| 298 | const twitterCard = 'summary' | ||
| 299 | const schemaType = 'ProfilePage' | ||
| 300 | |||
| 301 | customHtml = await ClientHtml.addTags(customHtml, { | ||
| 302 | url, | ||
| 303 | originUrl, | ||
| 304 | escapedTitle: escapeHTML(title), | ||
| 305 | escapedSiteName: escapeHTML(siteName), | ||
| 306 | escapedTruncatedDescription, | ||
| 307 | image, | ||
| 308 | ogType, | ||
| 309 | twitterCard, | ||
| 310 | schemaType, | ||
| 311 | disallowIndexation: !entity.Actor.isOwned() | ||
| 312 | }, {}) | ||
| 313 | |||
| 314 | return customHtml | ||
| 315 | } | ||
| 316 | |||
| 317 | private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { | ||
| 318 | const path = ClientHtml.getIndexPath(req, res, paramLang) | ||
| 319 | if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | ||
| 320 | |||
| 321 | const buffer = await readFile(path) | ||
| 322 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() | ||
| 323 | |||
| 324 | let html = buffer.toString() | ||
| 325 | |||
| 326 | html = ClientHtml.addManifestContentHash(html) | ||
| 327 | html = ClientHtml.addFaviconContentHash(html) | ||
| 328 | html = ClientHtml.addLogoContentHash(html) | ||
| 329 | html = ClientHtml.addCustomCSS(html) | ||
| 330 | html = ClientHtml.addServerConfig(html, serverConfig) | ||
| 331 | html = await ClientHtml.addAsyncPluginCSS(html) | ||
| 332 | |||
| 333 | ClientHtml.htmlCache[path] = html | ||
| 334 | |||
| 335 | return html | ||
| 336 | } | ||
| 337 | |||
| 338 | private static getIndexPath (req: express.Request, res: express.Response, paramLang: string) { | ||
| 339 | let lang: string | ||
| 340 | |||
| 341 | // Check param lang validity | ||
| 342 | if (paramLang && is18nLocale(paramLang)) { | ||
| 343 | lang = paramLang | ||
| 344 | |||
| 345 | // Save locale in cookies | ||
| 346 | res.cookie('clientLanguage', lang, { | ||
| 347 | secure: WEBSERVER.SCHEME === 'https', | ||
| 348 | sameSite: 'none', | ||
| 349 | maxAge: 1000 * 3600 * 24 * 90 // 3 months | ||
| 350 | }) | ||
| 351 | |||
| 352 | } else if (req.cookies.clientLanguage && is18nLocale(req.cookies.clientLanguage)) { | ||
| 353 | lang = req.cookies.clientLanguage | ||
| 354 | } else { | ||
| 355 | lang = req.acceptsLanguages(POSSIBLE_LOCALES) || getDefaultLocale() | ||
| 356 | } | ||
| 357 | |||
| 358 | logger.debug( | ||
| 359 | 'Serving %s HTML language', buildFileLocale(lang), | ||
| 360 | { cookie: req.cookies?.clientLanguage, paramLang, acceptLanguage: req.headers['accept-language'] } | ||
| 361 | ) | ||
| 362 | |||
| 363 | return join(root(), 'client', 'dist', buildFileLocale(lang), 'index.html') | ||
| 364 | } | ||
| 365 | |||
| 366 | private static getEmbedPath () { | ||
| 367 | return join(root(), 'client', 'dist', 'standalone', 'videos', 'embed.html') | ||
| 368 | } | ||
| 369 | |||
| 370 | private static addManifestContentHash (htmlStringPage: string) { | ||
| 371 | return htmlStringPage.replace('[manifestContentHash]', FILES_CONTENT_HASH.MANIFEST) | ||
| 372 | } | ||
| 373 | |||
| 374 | private static addFaviconContentHash (htmlStringPage: string) { | ||
| 375 | return htmlStringPage.replace('[faviconContentHash]', FILES_CONTENT_HASH.FAVICON) | ||
| 376 | } | ||
| 377 | |||
| 378 | private static addLogoContentHash (htmlStringPage: string) { | ||
| 379 | return htmlStringPage.replace('[logoContentHash]', FILES_CONTENT_HASH.LOGO) | ||
| 380 | } | ||
| 381 | |||
| 382 | private static addTitleTag (htmlStringPage: string, title?: string) { | ||
| 383 | let text = title || CONFIG.INSTANCE.NAME | ||
| 384 | if (title) text += ` - ${CONFIG.INSTANCE.NAME}` | ||
| 385 | |||
| 386 | const titleTag = `<title>${escapeHTML(text)}</title>` | ||
| 387 | |||
| 388 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) | ||
| 389 | } | ||
| 390 | |||
| 391 | private static addDescriptionTag (htmlStringPage: string, escapedTruncatedDescription?: string) { | ||
| 392 | const content = escapedTruncatedDescription || escapeHTML(CONFIG.INSTANCE.SHORT_DESCRIPTION) | ||
| 393 | const descriptionTag = `<meta name="description" content="${content}" />` | ||
| 394 | |||
| 395 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) | ||
| 396 | } | ||
| 397 | |||
| 398 | private static addCustomCSS (htmlStringPage: string) { | ||
| 399 | const styleTag = `<style class="custom-css-style">${CONFIG.INSTANCE.CUSTOMIZATIONS.CSS}</style>` | ||
| 400 | |||
| 401 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) | ||
| 402 | } | ||
| 403 | |||
| 404 | private static addServerConfig (htmlStringPage: string, serverConfig: HTMLServerConfig) { | ||
| 405 | // Stringify the JSON object, and then stringify the string object so we can inject it into the HTML | ||
| 406 | const serverConfigString = JSON.stringify(JSON.stringify(serverConfig)) | ||
| 407 | const configScriptTag = `<script type="application/javascript">window.PeerTubeServerConfig = ${serverConfigString}</script>` | ||
| 408 | |||
| 409 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.SERVER_CONFIG, configScriptTag) | ||
| 410 | } | ||
| 411 | |||
| 412 | private static async addAsyncPluginCSS (htmlStringPage: string) { | ||
| 413 | if (!pathExists(PLUGIN_GLOBAL_CSS_PATH)) { | ||
| 414 | logger.info('Plugin Global CSS file is not available (generation may still be in progress), ignoring it.') | ||
| 415 | return htmlStringPage | ||
| 416 | } | ||
| 417 | |||
| 418 | let globalCSSContent: Buffer | ||
| 419 | |||
| 420 | try { | ||
| 421 | globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH) | ||
| 422 | } catch (err) { | ||
| 423 | logger.error('Error retrieving the Plugin Global CSS file, ignoring it.', { err }) | ||
| 424 | return htmlStringPage | ||
| 425 | } | ||
| 426 | |||
| 427 | if (globalCSSContent.byteLength === 0) return htmlStringPage | ||
| 428 | |||
| 429 | const fileHash = sha256(globalCSSContent) | ||
| 430 | const linkTag = `<link rel="stylesheet" href="/plugins/global.css?hash=${fileHash}" />` | ||
| 431 | |||
| 432 | return htmlStringPage.replace('</head>', linkTag + '</head>') | ||
| 433 | } | ||
| 434 | |||
| 435 | private static generateOpenGraphMetaTags (tags: Tags) { | ||
| 436 | const metaTags = { | ||
| 437 | 'og:type': tags.ogType, | ||
| 438 | 'og:site_name': tags.escapedSiteName, | ||
| 439 | 'og:title': tags.escapedTitle, | ||
| 440 | 'og:image': tags.image.url | ||
| 441 | } | ||
| 442 | |||
| 443 | if (tags.image.width && tags.image.height) { | ||
| 444 | metaTags['og:image:width'] = tags.image.width | ||
| 445 | metaTags['og:image:height'] = tags.image.height | ||
| 446 | } | ||
| 447 | |||
| 448 | metaTags['og:url'] = tags.url | ||
| 449 | metaTags['og:description'] = tags.escapedTruncatedDescription | ||
| 450 | |||
| 451 | if (tags.embed) { | ||
| 452 | metaTags['og:video:url'] = tags.embed.url | ||
| 453 | metaTags['og:video:secure_url'] = tags.embed.url | ||
| 454 | metaTags['og:video:type'] = 'text/html' | ||
| 455 | metaTags['og:video:width'] = EMBED_SIZE.width | ||
| 456 | metaTags['og:video:height'] = EMBED_SIZE.height | ||
| 457 | } | ||
| 458 | |||
| 459 | return metaTags | ||
| 460 | } | ||
| 461 | |||
| 462 | private static generateStandardMetaTags (tags: Tags) { | ||
| 463 | return { | ||
| 464 | name: tags.escapedTitle, | ||
| 465 | description: tags.escapedTruncatedDescription, | ||
| 466 | image: tags.image.url | ||
| 467 | } | ||
| 468 | } | ||
| 469 | |||
| 470 | private static generateTwitterCardMetaTags (tags: Tags) { | ||
| 471 | const metaTags = { | ||
| 472 | 'twitter:card': tags.twitterCard, | ||
| 473 | 'twitter:site': CONFIG.SERVICES.TWITTER.USERNAME, | ||
| 474 | 'twitter:title': tags.escapedTitle, | ||
| 475 | 'twitter:description': tags.escapedTruncatedDescription, | ||
| 476 | 'twitter:image': tags.image.url | ||
| 477 | } | ||
| 478 | |||
| 479 | if (tags.image.width && tags.image.height) { | ||
| 480 | metaTags['twitter:image:width'] = tags.image.width | ||
| 481 | metaTags['twitter:image:height'] = tags.image.height | ||
| 482 | } | ||
| 483 | |||
| 484 | if (tags.twitterCard === 'player') { | ||
| 485 | metaTags['twitter:player'] = tags.embed.url | ||
| 486 | metaTags['twitter:player:width'] = EMBED_SIZE.width | ||
| 487 | metaTags['twitter:player:height'] = EMBED_SIZE.height | ||
| 488 | } | ||
| 489 | |||
| 490 | return metaTags | ||
| 491 | } | ||
| 492 | |||
| 493 | private static async generateSchemaTags (tags: Tags, context: HookContext) { | ||
| 494 | const schema = { | ||
| 495 | '@context': 'http://schema.org', | ||
| 496 | '@type': tags.schemaType, | ||
| 497 | 'name': tags.escapedTitle, | ||
| 498 | 'description': tags.escapedTruncatedDescription, | ||
| 499 | 'image': tags.image.url, | ||
| 500 | 'url': tags.url | ||
| 501 | } | ||
| 502 | |||
| 503 | if (tags.list) { | ||
| 504 | schema['numberOfItems'] = tags.list.numberOfItems | ||
| 505 | schema['thumbnailUrl'] = tags.image.url | ||
| 506 | } | ||
| 507 | |||
| 508 | if (tags.embed) { | ||
| 509 | schema['embedUrl'] = tags.embed.url | ||
| 510 | schema['uploadDate'] = tags.embed.createdAt | ||
| 511 | |||
| 512 | if (tags.embed.duration) schema['duration'] = tags.embed.duration | ||
| 513 | |||
| 514 | schema['thumbnailUrl'] = tags.image.url | ||
| 515 | schema['contentUrl'] = tags.url | ||
| 516 | } | ||
| 517 | |||
| 518 | return Hooks.wrapObject(schema, 'filter:html.client.json-ld.result', context) | ||
| 519 | } | ||
| 520 | |||
| 521 | private static async addTags (htmlStringPage: string, tagsValues: Tags, context: HookContext) { | ||
| 522 | const openGraphMetaTags = this.generateOpenGraphMetaTags(tagsValues) | ||
| 523 | const standardMetaTags = this.generateStandardMetaTags(tagsValues) | ||
| 524 | const twitterCardMetaTags = this.generateTwitterCardMetaTags(tagsValues) | ||
| 525 | const schemaTags = await this.generateSchemaTags(tagsValues, context) | ||
| 526 | |||
| 527 | const { url, escapedTitle, embed, originUrl, disallowIndexation } = tagsValues | ||
| 528 | |||
| 529 | const oembedLinkTags: { type: string, href: string, escapedTitle: string }[] = [] | ||
| 530 | |||
| 531 | if (embed) { | ||
| 532 | oembedLinkTags.push({ | ||
| 533 | type: 'application/json+oembed', | ||
| 534 | href: WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(url), | ||
| 535 | escapedTitle | ||
| 536 | }) | ||
| 537 | } | ||
| 538 | |||
| 539 | let tagsStr = '' | ||
| 540 | |||
| 541 | // Opengraph | ||
| 542 | Object.keys(openGraphMetaTags).forEach(tagName => { | ||
| 543 | const tagValue = openGraphMetaTags[tagName] | ||
| 544 | |||
| 545 | tagsStr += `<meta property="${tagName}" content="${tagValue}" />` | ||
| 546 | }) | ||
| 547 | |||
| 548 | // Standard | ||
| 549 | Object.keys(standardMetaTags).forEach(tagName => { | ||
| 550 | const tagValue = standardMetaTags[tagName] | ||
| 551 | |||
| 552 | tagsStr += `<meta property="${tagName}" content="${tagValue}" />` | ||
| 553 | }) | ||
| 554 | |||
| 555 | // Twitter card | ||
| 556 | Object.keys(twitterCardMetaTags).forEach(tagName => { | ||
| 557 | const tagValue = twitterCardMetaTags[tagName] | ||
| 558 | |||
| 559 | tagsStr += `<meta property="${tagName}" content="${tagValue}" />` | ||
| 560 | }) | ||
| 561 | |||
| 562 | // OEmbed | ||
| 563 | for (const oembedLinkTag of oembedLinkTags) { | ||
| 564 | tagsStr += `<link rel="alternate" type="${oembedLinkTag.type}" href="${oembedLinkTag.href}" title="${oembedLinkTag.escapedTitle}" />` | ||
| 565 | } | ||
| 566 | |||
| 567 | // Schema.org | ||
| 568 | if (schemaTags) { | ||
| 569 | tagsStr += `<script type="application/ld+json">${JSON.stringify(schemaTags)}</script>` | ||
| 570 | } | ||
| 571 | |||
| 572 | // SEO, use origin URL | ||
| 573 | tagsStr += `<link rel="canonical" href="${originUrl}" />` | ||
| 574 | |||
| 575 | if (disallowIndexation) { | ||
| 576 | tagsStr += `<meta name="robots" content="noindex" />` | ||
| 577 | } | ||
| 578 | |||
| 579 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.META_TAGS, tagsStr) | ||
| 580 | } | ||
| 581 | } | ||
| 582 | |||
| 583 | function sendHTML (html: string, res: express.Response, localizedHTML: boolean = false) { | ||
| 584 | res.set('Content-Type', 'text/html; charset=UTF-8') | ||
| 585 | |||
| 586 | if (localizedHTML) { | ||
| 587 | res.set('Vary', 'Accept-Language') | ||
| 588 | } | ||
| 589 | |||
| 590 | return res.send(html) | ||
| 591 | } | ||
| 592 | |||
| 593 | async function serveIndexHTML (req: express.Request, res: express.Response) { | ||
| 594 | if (req.accepts(ACCEPT_HEADERS) === 'html' || !req.headers.accept) { | ||
| 595 | try { | ||
| 596 | await generateHTMLPage(req, res, req.params.language) | ||
| 597 | return | ||
| 598 | } catch (err) { | ||
| 599 | logger.error('Cannot generate HTML page.', { err }) | ||
| 600 | return res.status(HttpStatusCode.INTERNAL_SERVER_ERROR_500).end() | ||
| 601 | } | ||
| 602 | } | ||
| 603 | |||
| 604 | return res.status(HttpStatusCode.NOT_ACCEPTABLE_406).end() | ||
| 605 | } | ||
| 606 | |||
| 607 | // --------------------------------------------------------------------------- | ||
| 608 | |||
| 609 | export { | ||
| 610 | ClientHtml, | ||
| 611 | sendHTML, | ||
| 612 | serveIndexHTML | ||
| 613 | } | ||
| 614 | |||
| 615 | async function generateHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { | ||
| 616 | const html = await ClientHtml.getDefaultHTMLPage(req, res, paramLang) | ||
| 617 | |||
| 618 | return sendHTML(html, res, true) | ||
| 619 | } | ||
| 620 | |||
| 621 | function buildEscapedTruncatedDescription (description: string) { | ||
| 622 | return truncate(mdToOneLinePlainText(description), { length: 200 }) | ||
| 623 | } | ||
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts deleted file mode 100644 index f5c3e4745..000000000 --- a/server/lib/emailer.ts +++ /dev/null | |||
| @@ -1,284 +0,0 @@ | |||
| 1 | import { readFileSync } from 'fs-extra' | ||
| 2 | import { merge } from 'lodash' | ||
| 3 | import { createTransport, Transporter } from 'nodemailer' | ||
| 4 | import { join } from 'path' | ||
| 5 | import { arrayify, root } from '@shared/core-utils' | ||
| 6 | import { EmailPayload, UserRegistrationState } from '@shared/models' | ||
| 7 | import { SendEmailDefaultOptions } from '../../shared/models/server/emailer.model' | ||
| 8 | import { isTestOrDevInstance } from '../helpers/core-utils' | ||
| 9 | import { bunyanLogger, logger } from '../helpers/logger' | ||
| 10 | import { CONFIG, isEmailEnabled } from '../initializers/config' | ||
| 11 | import { WEBSERVER } from '../initializers/constants' | ||
| 12 | import { MRegistration, MUser } from '../types/models' | ||
| 13 | import { JobQueue } from './job-queue' | ||
| 14 | |||
| 15 | const Email = require('email-templates') | ||
| 16 | |||
| 17 | class Emailer { | ||
| 18 | |||
| 19 | private static instance: Emailer | ||
| 20 | private initialized = false | ||
| 21 | private transporter: Transporter | ||
| 22 | |||
| 23 | private constructor () { | ||
| 24 | } | ||
| 25 | |||
| 26 | init () { | ||
| 27 | // Already initialized | ||
| 28 | if (this.initialized === true) return | ||
| 29 | this.initialized = true | ||
| 30 | |||
| 31 | if (!isEmailEnabled()) { | ||
| 32 | if (!isTestOrDevInstance()) { | ||
| 33 | logger.error('Cannot use SMTP server because of lack of configuration. PeerTube will not be able to send mails!') | ||
| 34 | } | ||
| 35 | |||
| 36 | return | ||
| 37 | } | ||
| 38 | |||
| 39 | if (CONFIG.SMTP.TRANSPORT === 'smtp') this.initSMTPTransport() | ||
| 40 | else if (CONFIG.SMTP.TRANSPORT === 'sendmail') this.initSendmailTransport() | ||
| 41 | } | ||
| 42 | |||
| 43 | async checkConnection () { | ||
| 44 | if (!this.transporter || CONFIG.SMTP.TRANSPORT !== 'smtp') return | ||
| 45 | |||
| 46 | logger.info('Testing SMTP server...') | ||
| 47 | |||
| 48 | try { | ||
| 49 | const success = await this.transporter.verify() | ||
| 50 | if (success !== true) this.warnOnConnectionFailure() | ||
| 51 | |||
| 52 | logger.info('Successfully connected to SMTP server.') | ||
| 53 | } catch (err) { | ||
| 54 | this.warnOnConnectionFailure(err) | ||
| 55 | } | ||
| 56 | } | ||
| 57 | |||
| 58 | addPasswordResetEmailJob (username: string, to: string, resetPasswordUrl: string) { | ||
| 59 | const emailPayload: EmailPayload = { | ||
| 60 | template: 'password-reset', | ||
| 61 | to: [ to ], | ||
| 62 | subject: 'Reset your account password', | ||
| 63 | locals: { | ||
| 64 | username, | ||
| 65 | resetPasswordUrl, | ||
| 66 | |||
| 67 | hideNotificationPreferencesLink: true | ||
| 68 | } | ||
| 69 | } | ||
| 70 | |||
| 71 | return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) | ||
| 72 | } | ||
| 73 | |||
| 74 | addPasswordCreateEmailJob (username: string, to: string, createPasswordUrl: string) { | ||
| 75 | const emailPayload: EmailPayload = { | ||
| 76 | template: 'password-create', | ||
| 77 | to: [ to ], | ||
| 78 | subject: 'Create your account password', | ||
| 79 | locals: { | ||
| 80 | username, | ||
| 81 | createPasswordUrl, | ||
| 82 | |||
| 83 | hideNotificationPreferencesLink: true | ||
| 84 | } | ||
| 85 | } | ||
| 86 | |||
| 87 | return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) | ||
| 88 | } | ||
| 89 | |||
| 90 | addVerifyEmailJob (options: { | ||
| 91 | username: string | ||
| 92 | isRegistrationRequest: boolean | ||
| 93 | to: string | ||
| 94 | verifyEmailUrl: string | ||
| 95 | }) { | ||
| 96 | const { username, isRegistrationRequest, to, verifyEmailUrl } = options | ||
| 97 | |||
| 98 | const emailPayload: EmailPayload = { | ||
| 99 | template: 'verify-email', | ||
| 100 | to: [ to ], | ||
| 101 | subject: `Verify your email on ${CONFIG.INSTANCE.NAME}`, | ||
| 102 | locals: { | ||
| 103 | username, | ||
| 104 | verifyEmailUrl, | ||
| 105 | isRegistrationRequest, | ||
| 106 | |||
| 107 | hideNotificationPreferencesLink: true | ||
| 108 | } | ||
| 109 | } | ||
| 110 | |||
| 111 | return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) | ||
| 112 | } | ||
| 113 | |||
| 114 | addUserBlockJob (user: MUser, blocked: boolean, reason?: string) { | ||
| 115 | const reasonString = reason ? ` for the following reason: ${reason}` : '' | ||
| 116 | const blockedWord = blocked ? 'blocked' : 'unblocked' | ||
| 117 | |||
| 118 | const to = user.email | ||
| 119 | const emailPayload: EmailPayload = { | ||
| 120 | to: [ to ], | ||
| 121 | subject: 'Account ' + blockedWord, | ||
| 122 | text: `Your account ${user.username} on ${CONFIG.INSTANCE.NAME} has been ${blockedWord}${reasonString}.` | ||
| 123 | } | ||
| 124 | |||
| 125 | return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) | ||
| 126 | } | ||
| 127 | |||
| 128 | addContactFormJob (fromEmail: string, fromName: string, subject: string, body: string) { | ||
| 129 | const emailPayload: EmailPayload = { | ||
| 130 | template: 'contact-form', | ||
| 131 | to: [ CONFIG.ADMIN.EMAIL ], | ||
| 132 | replyTo: `"${fromName}" <${fromEmail}>`, | ||
| 133 | subject: `(contact form) ${subject}`, | ||
| 134 | locals: { | ||
| 135 | fromName, | ||
| 136 | fromEmail, | ||
| 137 | body, | ||
| 138 | |||
| 139 | // There are not notification preferences for the contact form | ||
| 140 | hideNotificationPreferencesLink: true | ||
| 141 | } | ||
| 142 | } | ||
| 143 | |||
| 144 | return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) | ||
| 145 | } | ||
| 146 | |||
| 147 | addUserRegistrationRequestProcessedJob (registration: MRegistration) { | ||
| 148 | let template: string | ||
| 149 | let subject: string | ||
| 150 | if (registration.state === UserRegistrationState.ACCEPTED) { | ||
| 151 | template = 'user-registration-request-accepted' | ||
| 152 | subject = `Your registration request for ${registration.username} has been accepted` | ||
| 153 | } else { | ||
| 154 | template = 'user-registration-request-rejected' | ||
| 155 | subject = `Your registration request for ${registration.username} has been rejected` | ||
| 156 | } | ||
| 157 | |||
| 158 | const to = registration.email | ||
| 159 | const emailPayload: EmailPayload = { | ||
| 160 | to: [ to ], | ||
| 161 | template, | ||
| 162 | subject, | ||
| 163 | locals: { | ||
| 164 | username: registration.username, | ||
| 165 | moderationResponse: registration.moderationResponse, | ||
| 166 | loginLink: WEBSERVER.URL + '/login' | ||
| 167 | } | ||
| 168 | } | ||
| 169 | |||
| 170 | return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) | ||
| 171 | } | ||
| 172 | |||
| 173 | async sendMail (options: EmailPayload) { | ||
| 174 | if (!isEmailEnabled()) { | ||
| 175 | logger.info('Cannot send mail because SMTP is not configured.') | ||
| 176 | return | ||
| 177 | } | ||
| 178 | |||
| 179 | const fromDisplayName = options.from | ||
| 180 | ? options.from | ||
| 181 | : CONFIG.INSTANCE.NAME | ||
| 182 | |||
| 183 | const email = new Email({ | ||
| 184 | send: true, | ||
| 185 | htmlToText: { | ||
| 186 | selectors: [ | ||
| 187 | { selector: 'img', format: 'skip' }, | ||
| 188 | { selector: 'a', options: { hideLinkHrefIfSameAsText: true } } | ||
| 189 | ] | ||
| 190 | }, | ||
| 191 | message: { | ||
| 192 | from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>` | ||
| 193 | }, | ||
| 194 | transport: this.transporter, | ||
| 195 | views: { | ||
| 196 | root: join(root(), 'dist', 'server', 'lib', 'emails') | ||
| 197 | }, | ||
| 198 | subjectPrefix: CONFIG.EMAIL.SUBJECT.PREFIX | ||
| 199 | }) | ||
| 200 | |||
| 201 | const toEmails = arrayify(options.to) | ||
| 202 | |||
| 203 | for (const to of toEmails) { | ||
| 204 | const baseOptions: SendEmailDefaultOptions = { | ||
| 205 | template: 'common', | ||
| 206 | message: { | ||
| 207 | to, | ||
| 208 | from: options.from, | ||
| 209 | subject: options.subject, | ||
| 210 | replyTo: options.replyTo | ||
| 211 | }, | ||
| 212 | locals: { // default variables available in all templates | ||
| 213 | WEBSERVER, | ||
| 214 | EMAIL: CONFIG.EMAIL, | ||
| 215 | instanceName: CONFIG.INSTANCE.NAME, | ||
| 216 | text: options.text, | ||
| 217 | subject: options.subject | ||
| 218 | } | ||
| 219 | } | ||
| 220 | |||
| 221 | // overridden/new variables given for a specific template in the payload | ||
| 222 | const sendOptions = merge(baseOptions, options) | ||
| 223 | |||
| 224 | await email.send(sendOptions) | ||
| 225 | .then(res => logger.debug('Sent email.', { res })) | ||
| 226 | .catch(err => logger.error('Error in email sender.', { err })) | ||
| 227 | } | ||
| 228 | } | ||
| 229 | |||
| 230 | private warnOnConnectionFailure (err?: Error) { | ||
| 231 | logger.error('Failed to connect to SMTP %s:%d.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT, { err }) | ||
| 232 | } | ||
| 233 | |||
| 234 | private initSMTPTransport () { | ||
| 235 | logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) | ||
| 236 | |||
| 237 | let tls | ||
| 238 | if (CONFIG.SMTP.CA_FILE) { | ||
| 239 | tls = { | ||
| 240 | ca: [ readFileSync(CONFIG.SMTP.CA_FILE) ] | ||
| 241 | } | ||
| 242 | } | ||
| 243 | |||
| 244 | let auth | ||
| 245 | if (CONFIG.SMTP.USERNAME && CONFIG.SMTP.PASSWORD) { | ||
| 246 | auth = { | ||
| 247 | user: CONFIG.SMTP.USERNAME, | ||
| 248 | pass: CONFIG.SMTP.PASSWORD | ||
| 249 | } | ||
| 250 | } | ||
| 251 | |||
| 252 | this.transporter = createTransport({ | ||
| 253 | host: CONFIG.SMTP.HOSTNAME, | ||
| 254 | port: CONFIG.SMTP.PORT, | ||
| 255 | secure: CONFIG.SMTP.TLS, | ||
| 256 | debug: CONFIG.LOG.LEVEL === 'debug', | ||
| 257 | logger: bunyanLogger as any, | ||
| 258 | ignoreTLS: CONFIG.SMTP.DISABLE_STARTTLS, | ||
| 259 | tls, | ||
| 260 | auth | ||
| 261 | }) | ||
| 262 | } | ||
| 263 | |||
| 264 | private initSendmailTransport () { | ||
| 265 | logger.info('Using sendmail to send emails') | ||
| 266 | |||
| 267 | this.transporter = createTransport({ | ||
| 268 | sendmail: true, | ||
| 269 | newline: 'unix', | ||
| 270 | path: CONFIG.SMTP.SENDMAIL, | ||
| 271 | logger: bunyanLogger | ||
| 272 | }) | ||
| 273 | } | ||
| 274 | |||
| 275 | static get Instance () { | ||
| 276 | return this.instance || (this.instance = new this()) | ||
| 277 | } | ||
| 278 | } | ||
| 279 | |||
| 280 | // --------------------------------------------------------------------------- | ||
| 281 | |||
| 282 | export { | ||
| 283 | Emailer | ||
| 284 | } | ||
diff --git a/server/lib/emails/abuse-new-message/html.pug b/server/lib/emails/abuse-new-message/html.pug deleted file mode 100644 index c1d452e7d..000000000 --- a/server/lib/emails/abuse-new-message/html.pug +++ /dev/null | |||
| @@ -1,11 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | include ../common/mixins.pug | ||
| 3 | |||
| 4 | block title | ||
| 5 | | New message on abuse report | ||
| 6 | |||
| 7 | block content | ||
| 8 | p | ||
| 9 | | A new message by #{messageAccountName} was posted on #[a(href=abuseUrl) abuse report ##{abuseId}] on #{instanceName} | ||
| 10 | blockquote #{messageText} | ||
| 11 | br(style="display: none;") | ||
diff --git a/server/lib/emails/abuse-state-change/html.pug b/server/lib/emails/abuse-state-change/html.pug deleted file mode 100644 index bb243e729..000000000 --- a/server/lib/emails/abuse-state-change/html.pug +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | include ../common/mixins.pug | ||
| 3 | |||
| 4 | block title | ||
| 5 | | Abuse report state changed | ||
| 6 | |||
| 7 | block content | ||
| 8 | p | ||
| 9 | | #[a(href=abuseUrl) Your abuse report ##{abuseId}] on #{instanceName} has been #{isAccepted ? 'accepted' : 'rejected'} | ||
diff --git a/server/lib/emails/account-abuse-new/html.pug b/server/lib/emails/account-abuse-new/html.pug deleted file mode 100644 index e4c0366fb..000000000 --- a/server/lib/emails/account-abuse-new/html.pug +++ /dev/null | |||
| @@ -1,14 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | include ../common/mixins.pug | ||
| 3 | |||
| 4 | block title | ||
| 5 | | An account is pending moderation | ||
| 6 | |||
| 7 | block content | ||
| 8 | p | ||
| 9 | | #[a(href=WEBSERVER.URL) #{instanceName}] received an abuse report for the #{isLocal ? '' : 'remote '}account | ||
| 10 | a(href=accountUrl) #{accountDisplayName} | ||
| 11 | |||
| 12 | p The reporter, #{reporter}, cited the following reason(s): | ||
| 13 | blockquote #{reason} | ||
| 14 | br(style="display: none;") | ||
diff --git a/server/lib/emails/common/base.pug b/server/lib/emails/common/base.pug deleted file mode 100644 index 41e94564d..000000000 --- a/server/lib/emails/common/base.pug +++ /dev/null | |||
| @@ -1,258 +0,0 @@ | |||
| 1 | //- | ||
| 2 | The email background color is defined in three places: | ||
| 3 | 1. body tag: for most email clients | ||
| 4 | 2. center tag: for Gmail and Inbox mobile apps and web versions of Gmail, GSuite, Inbox, Yahoo, AOL, Libero, Comcast, freenet, Mail.ru, Orange.fr | ||
| 5 | 3. mso conditional: For Windows 10 Mail | ||
| 6 | - var backgroundColor = "#fff"; | ||
| 7 | - var mainColor = "#f2690d"; | ||
| 8 | doctype html | ||
| 9 | head | ||
| 10 | // This template is heavily adapted from the Cerberus Fluid template. Kudos to them! | ||
| 11 | meta(charset='utf-8') | ||
| 12 | //- utf-8 works for most cases | ||
| 13 | meta(name='viewport' content='width=device-width') | ||
| 14 | //- Forcing initial-scale shouldn't be necessary | ||
| 15 | meta(http-equiv='X-UA-Compatible' content='IE=edge') | ||
| 16 | //- Use the latest (edge) version of IE rendering engine | ||
| 17 | meta(name='x-apple-disable-message-reformatting') | ||
| 18 | //- Disable auto-scale in iOS 10 Mail entirely | ||
| 19 | meta(name='format-detection' content='telephone=no,address=no,email=no,date=no,url=no') | ||
| 20 | //- Tell iOS not to automatically link certain text strings. | ||
| 21 | meta(name='color-scheme' content='light') | ||
| 22 | meta(name='supported-color-schemes' content='light') | ||
| 23 | //- The title tag shows in email notifications, like Android 4.4. | ||
| 24 | title #{subject} | ||
| 25 | //- What it does: Makes background images in 72ppi Outlook render at correct size. | ||
| 26 | //if gte mso 9 | ||
| 27 | xml | ||
| 28 | o:officedocumentsettings | ||
| 29 | o:allowpng | ||
| 30 | o:pixelsperinch 96 | ||
| 31 | //- CSS Reset : BEGIN | ||
| 32 | style. | ||
| 33 | /* What it does: Tells the email client that only light styles are provided but the client can transform them to dark. A duplicate of meta color-scheme meta tag above. */ | ||
| 34 | :root { | ||
| 35 | color-scheme: light; | ||
| 36 | supported-color-schemes: light; | ||
| 37 | } | ||
| 38 | /* What it does: Remove spaces around the email design added by some email clients. */ | ||
| 39 | /* Beware: It can remove the padding / margin and add a background color to the compose a reply window. */ | ||
| 40 | html, | ||
| 41 | body { | ||
| 42 | margin: 0 auto !important; | ||
| 43 | padding: 0 !important; | ||
| 44 | height: 100% !important; | ||
| 45 | width: 100% !important; | ||
| 46 | } | ||
| 47 | /* What it does: Stops email clients resizing small text. */ | ||
| 48 | * { | ||
| 49 | -ms-text-size-adjust: 100%; | ||
| 50 | -webkit-text-size-adjust: 100%; | ||
| 51 | } | ||
| 52 | /* What it does: Centers email on Android 4.4 */ | ||
| 53 | div[style*="margin: 16px 0"] { | ||
| 54 | margin: 0 !important; | ||
| 55 | } | ||
| 56 | /* What it does: forces Samsung Android mail clients to use the entire viewport */ | ||
| 57 | #MessageViewBody, #MessageWebViewDiv{ | ||
| 58 | width: 100% !important; | ||
| 59 | } | ||
| 60 | /* What it does: Stops Outlook from adding extra spacing to tables. */ | ||
| 61 | table, | ||
| 62 | td { | ||
| 63 | mso-table-lspace: 0pt !important; | ||
| 64 | mso-table-rspace: 0pt !important; | ||
| 65 | } | ||
| 66 | /* What it does: Fixes webkit padding issue. */ | ||
| 67 | table { | ||
| 68 | border-spacing: 0 !important; | ||
| 69 | border-collapse: collapse !important; | ||
| 70 | table-layout: fixed !important; | ||
| 71 | margin: 0 auto !important; | ||
| 72 | } | ||
| 73 | /* What it does: Uses a better rendering method when resizing images in IE. */ | ||
| 74 | img { | ||
| 75 | -ms-interpolation-mode:bicubic; | ||
| 76 | } | ||
| 77 | /* What it does: Prevents Windows 10 Mail from underlining links despite inline CSS. Styles for underlined links should be inline. */ | ||
| 78 | a { | ||
| 79 | text-decoration: none; | ||
| 80 | } | ||
| 81 | a:not(.nocolor) { | ||
| 82 | color: #{mainColor}; | ||
| 83 | } | ||
| 84 | a.nocolor { | ||
| 85 | color: inherit !important; | ||
| 86 | } | ||
| 87 | /* What it does: A work-around for email clients meddling in triggered links. */ | ||
| 88 | a[x-apple-data-detectors], /* iOS */ | ||
| 89 | .unstyle-auto-detected-links a, | ||
| 90 | .aBn { | ||
| 91 | border-bottom: 0 !important; | ||
| 92 | cursor: default !important; | ||
| 93 | color: inherit !important; | ||
| 94 | text-decoration: none !important; | ||
| 95 | font-size: inherit !important; | ||
| 96 | font-family: inherit !important; | ||
| 97 | font-weight: inherit !important; | ||
| 98 | line-height: inherit !important; | ||
| 99 | } | ||
| 100 | /* What it does: Prevents Gmail from displaying a download button on large, non-linked images. */ | ||
| 101 | .a6S { | ||
| 102 | display: none !important; | ||
| 103 | opacity: 0.01 !important; | ||
| 104 | } | ||
| 105 | /* What it does: Prevents Gmail from changing the text color in conversation threads. */ | ||
| 106 | .im { | ||
| 107 | color: inherit !important; | ||
| 108 | } | ||
| 109 | /* If the above doesn't work, add a .g-img class to any image in question. */ | ||
| 110 | img.g-img + div { | ||
| 111 | display: none !important; | ||
| 112 | } | ||
| 113 | /* What it does: Removes right gutter in Gmail iOS app: https://github.com/TedGoas/Cerberus/issues/89 */ | ||
| 114 | /* Create one of these media queries for each additional viewport size you'd like to fix */ | ||
| 115 | /* iPhone 4, 4S, 5, 5S, 5C, and 5SE */ | ||
| 116 | @media only screen and (min-device-width: 320px) and (max-device-width: 374px) { | ||
| 117 | u ~ div .email-container { | ||
| 118 | min-width: 320px !important; | ||
| 119 | } | ||
| 120 | } | ||
| 121 | /* iPhone 6, 6S, 7, 8, and X */ | ||
| 122 | @media only screen and (min-device-width: 375px) and (max-device-width: 413px) { | ||
| 123 | u ~ div .email-container { | ||
| 124 | min-width: 375px !important; | ||
| 125 | } | ||
| 126 | } | ||
| 127 | /* iPhone 6+, 7+, and 8+ */ | ||
| 128 | @media only screen and (min-device-width: 414px) { | ||
| 129 | u ~ div .email-container { | ||
| 130 | min-width: 414px !important; | ||
| 131 | } | ||
| 132 | } | ||
| 133 | //- CSS Reset : END | ||
| 134 | //- CSS for PeerTube : START | ||
| 135 | style. | ||
| 136 | blockquote { | ||
| 137 | margin-left: 0; | ||
| 138 | padding-left: 20px; | ||
| 139 | border-left: 2px solid #f2690d; | ||
| 140 | } | ||
| 141 | //- CSS for PeerTube : END | ||
| 142 | //- Progressive Enhancements : BEGIN | ||
| 143 | style. | ||
| 144 | /* What it does: Hover styles for buttons */ | ||
| 145 | .button-td, | ||
| 146 | .button-a { | ||
| 147 | transition: all 100ms ease-in; | ||
| 148 | } | ||
| 149 | .button-td-primary:hover, | ||
| 150 | .button-a-primary:hover { | ||
| 151 | background: #555555 !important; | ||
| 152 | border-color: #555555 !important; | ||
| 153 | } | ||
| 154 | /* Media Queries */ | ||
| 155 | @media screen and (max-width: 600px) { | ||
| 156 | /* What it does: Adjust typography on small screens to improve readability */ | ||
| 157 | .email-container p { | ||
| 158 | font-size: 17px !important; | ||
| 159 | } | ||
| 160 | } | ||
| 161 | //- Progressive Enhancements : END | ||
| 162 | |||
| 163 | body(width="100%" style="margin: 0; padding: 0 !important; mso-line-height-rule: exactly; background-color: #{backgroundColor};") | ||
| 164 | center(role='article' aria-roledescription='email' lang='en' style='width: 100%; background-color: #{backgroundColor};') | ||
| 165 | //if mso | IE | ||
| 166 | table(role='presentation' border='0' cellpadding='0' cellspacing='0' width='100%' style='background-color: #fff;') | ||
| 167 | tr | ||
| 168 | td | ||
| 169 | //- Visually Hidden Preheader Text : BEGIN | ||
| 170 | div(style='max-height:0; overflow:hidden; mso-hide:all;' aria-hidden='true') | ||
| 171 | block preheader | ||
| 172 | //- Visually Hidden Preheader Text : END | ||
| 173 | |||
| 174 | //- Create white space after the desired preview text so email clients don’t pull other distracting text into the inbox preview. Extend as necessary. | ||
| 175 | //- Preview Text Spacing Hack : BEGIN | ||
| 176 | div(style='display: none; font-size: 1px; line-height: 1px; max-height: 0px; max-width: 0px; opacity: 0; overflow: hidden; mso-hide: all; font-family: sans-serif;') | ||
| 177 | | ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ | ||
| 178 | //- Preview Text Spacing Hack : END | ||
| 179 | |||
| 180 | //- | ||
| 181 | Set the email width. Defined in two places: | ||
| 182 | 1. max-width for all clients except Desktop Windows Outlook, allowing the email to squish on narrow but never go wider than 600px. | ||
| 183 | 2. MSO tags for Desktop Windows Outlook enforce a 600px width. | ||
| 184 | .email-container(style='max-width: 600px; margin: 0 auto;') | ||
| 185 | //if mso | ||
| 186 | table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' width='600') | ||
| 187 | tr | ||
| 188 | td | ||
| 189 | //- Email Body : BEGIN | ||
| 190 | table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%' style='margin: auto;') | ||
| 191 | //- 1 Column Text + Button : BEGIN | ||
| 192 | tr | ||
| 193 | td(style='background-color: #ffffff;') | ||
| 194 | table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%') | ||
| 195 | tr | ||
| 196 | td(style='padding: 20px; font-family: sans-serif; font-size: 15px; line-height: 20px; color: #555555;') | ||
| 197 | table(role="presentation" border="0" cellpadding="0" cellspacing="0" width="100%") | ||
| 198 | tr | ||
| 199 | td(width="40px") | ||
| 200 | img(src=`${WEBSERVER.URL}/client/assets/images/icons/icon-192x192.png` width="auto" height="30px" alt="" border="0" style="height: 30px; background: #ffffff; font-family: sans-serif; font-size: 15px; line-height: 15px; color: #555555;") | ||
| 201 | td | ||
| 202 | h1(style='margin: 10px 0 10px 0; font-family: sans-serif; font-size: 25px; line-height: 30px; color: #333333; font-weight: normal;') | ||
| 203 | block title | ||
| 204 | if title | ||
| 205 | | #{title} | ||
| 206 | else | ||
| 207 | | Something requires your attention | ||
| 208 | p(style='margin: 0;') | ||
| 209 | block body | ||
| 210 | if action | ||
| 211 | tr | ||
| 212 | td(style='padding: 0 20px;') | ||
| 213 | //- Button : BEGIN | ||
| 214 | table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' style='margin: auto;') | ||
| 215 | tr | ||
| 216 | td.button-td.button-td-primary(style='border-radius: 4px; background: #222222;') | ||
| 217 | a.button-a.button-a-primary(href=action.url style='background: #222222; border: 1px solid #000000; font-family: sans-serif; font-size: 15px; line-height: 15px; text-decoration: none; padding: 13px 17px; color: #ffffff; display: block; border-radius: 4px;') #{action.text} | ||
| 218 | //- Button : END | ||
| 219 | //- 1 Column Text + Button : END | ||
| 220 | //- Clear Spacer : BEGIN | ||
| 221 | tr | ||
| 222 | td(aria-hidden='true' height='20' style='font-size: 0px; line-height: 0px;') | ||
| 223 | br | ||
| 224 | //- Clear Spacer : END | ||
| 225 | //- Email Body : END | ||
| 226 | //- Email Footer : BEGIN | ||
| 227 | unless hideNotificationPreferencesLink | ||
| 228 | table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%' style='margin: auto;') | ||
| 229 | tr | ||
| 230 | td(style='padding: 20px; padding-bottom: 0px; font-family: sans-serif; font-size: 12px; line-height: 15px; text-align: center; color: #888888;') | ||
| 231 | webversion | ||
| 232 | a.nocolor(href=`${WEBSERVER.URL}/my-account/notifications` style='color: #cccccc; font-weight: bold;') View in your notifications | ||
| 233 | br | ||
| 234 | tr | ||
| 235 | td(style='padding: 20px; padding-top: 10px; font-family: sans-serif; font-size: 12px; line-height: 15px; text-align: center; color: #888888;') | ||
| 236 | unsubscribe | ||
| 237 | a.nocolor(href=`${WEBSERVER.URL}/my-account/settings#notifications` style='color: #888888;') Manage your notification preferences in your profile | ||
| 238 | br | ||
| 239 | //- Email Footer : END | ||
| 240 | //if mso | ||
| 241 | //- Full Bleed Background Section : BEGIN | ||
| 242 | table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%' style=`background-color: ${mainColor};`) | ||
| 243 | tr | ||
| 244 | td | ||
| 245 | .email-container(align='center' style='max-width: 600px; margin: auto;') | ||
| 246 | //if mso | ||
| 247 | table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='600' align='center') | ||
| 248 | tr | ||
| 249 | td | ||
| 250 | table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%') | ||
| 251 | tr | ||
| 252 | td(style='padding: 20px; text-align: left; font-family: sans-serif; font-size: 12px; line-height: 20px; color: #ffffff;') | ||
| 253 | table(role="presentation" border="0" cellpadding="0" cellspacing="0" width="100%") | ||
| 254 | tr | ||
| 255 | td(valign="top") #[a(href="https://github.com/Chocobozzz/PeerTube" style="color: white !important") PeerTube © 2015-#{new Date().getFullYear()}] #[a(href="https://github.com/Chocobozzz/PeerTube/blob/master/CREDITS.md" style="color: white !important") PeerTube Contributors] | ||
| 256 | //if mso | ||
| 257 | //- Full Bleed Background Section : END | ||
| 258 | //if mso | IE | ||
diff --git a/server/lib/emails/common/greetings.pug b/server/lib/emails/common/greetings.pug deleted file mode 100644 index 5efe29dfb..000000000 --- a/server/lib/emails/common/greetings.pug +++ /dev/null | |||
| @@ -1,11 +0,0 @@ | |||
| 1 | extends base | ||
| 2 | |||
| 3 | block body | ||
| 4 | if username | ||
| 5 | p Hi #{username}, | ||
| 6 | else | ||
| 7 | p Hi, | ||
| 8 | block content | ||
| 9 | p | ||
| 10 | | Cheers,#[br] | ||
| 11 | | #{EMAIL.BODY.SIGNATURE} \ No newline at end of file | ||
diff --git a/server/lib/emails/common/html.pug b/server/lib/emails/common/html.pug deleted file mode 100644 index d76168b85..000000000 --- a/server/lib/emails/common/html.pug +++ /dev/null | |||
| @@ -1,4 +0,0 @@ | |||
| 1 | extends greetings | ||
| 2 | |||
| 3 | block content | ||
| 4 | p !{text} \ No newline at end of file | ||
diff --git a/server/lib/emails/common/mixins.pug b/server/lib/emails/common/mixins.pug deleted file mode 100644 index 831211864..000000000 --- a/server/lib/emails/common/mixins.pug +++ /dev/null | |||
| @@ -1,7 +0,0 @@ | |||
| 1 | mixin channel(channel) | ||
| 2 | - var handle = `${channel.name}@${channel.host}` | ||
| 3 | | #[a(href=`${WEBSERVER.URL}/video-channels/${handle}` title=handle) #{channel.displayName}] | ||
| 4 | |||
| 5 | mixin account(account) | ||
| 6 | - var handle = `${account.name}@${account.host}` | ||
| 7 | | #[a(href=`${WEBSERVER.URL}/accounts/${handle}` title=handle) #{account.displayName}] | ||
diff --git a/server/lib/emails/contact-form/html.pug b/server/lib/emails/contact-form/html.pug deleted file mode 100644 index 5a24fa6f1..000000000 --- a/server/lib/emails/contact-form/html.pug +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Someone just used the contact form | ||
| 5 | |||
| 6 | block content | ||
| 7 | p #{fromName} sent you a message via the contact form on #[a(href=WEBSERVER.URL) #{instanceName}]: | ||
| 8 | blockquote(style='white-space: pre-wrap') #{body} | ||
| 9 | p You can contact them at #[a(href=`mailto:${fromEmail}`) #{fromEmail}], or simply reply to this email to get in touch. \ No newline at end of file | ||
diff --git a/server/lib/emails/follower-on-channel/html.pug b/server/lib/emails/follower-on-channel/html.pug deleted file mode 100644 index 8a352e90f..000000000 --- a/server/lib/emails/follower-on-channel/html.pug +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | New follower on your channel | ||
| 5 | |||
| 6 | block content | ||
| 7 | p. | ||
| 8 | Your #{followType} #[a(href=followingUrl) #{followingName}] has a new subscriber: | ||
| 9 | #[a(href=followerUrl) #{followerName}]. \ No newline at end of file | ||
diff --git a/server/lib/emails/password-create/html.pug b/server/lib/emails/password-create/html.pug deleted file mode 100644 index afa30ae97..000000000 --- a/server/lib/emails/password-create/html.pug +++ /dev/null | |||
| @@ -1,10 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Password creation for your account | ||
| 5 | |||
| 6 | block content | ||
| 7 | p. | ||
| 8 | Welcome to #[a(href=WEBSERVER.URL) #{instanceName}]. Your username is: #{username}. | ||
| 9 | Please set your password by following #[a(href=createPasswordUrl) this link]: #[a(href=createPasswordUrl) #{createPasswordUrl}] | ||
| 10 | (this link will expire within seven days). \ No newline at end of file | ||
diff --git a/server/lib/emails/password-reset/html.pug b/server/lib/emails/password-reset/html.pug deleted file mode 100644 index 2af2885bc..000000000 --- a/server/lib/emails/password-reset/html.pug +++ /dev/null | |||
| @@ -1,12 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Password reset for your account | ||
| 5 | |||
| 6 | block content | ||
| 7 | p. | ||
| 8 | A reset password procedure for your account #{username} has been requested on #[a(href=WEBSERVER.URL) #{instanceName}]. | ||
| 9 | Please follow #[a(href=resetPasswordUrl) this link] to reset it: #[a(href=resetPasswordUrl) #{resetPasswordUrl}] | ||
| 10 | (the link will expire within 1 hour). | ||
| 11 | p. | ||
| 12 | If you are not the person who initiated this request, please ignore this email. | ||
diff --git a/server/lib/emails/peertube-version-new/html.pug b/server/lib/emails/peertube-version-new/html.pug deleted file mode 100644 index 2f4d9399d..000000000 --- a/server/lib/emails/peertube-version-new/html.pug +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | New PeerTube version available | ||
| 5 | |||
| 6 | block content | ||
| 7 | p | ||
| 8 | | A new version of PeerTube is available: #{latestVersion}. | ||
| 9 | | You can check the latest news on #[a(href="https://joinpeertube.org/news") JoinPeerTube]. | ||
diff --git a/server/lib/emails/plugin-version-new/html.pug b/server/lib/emails/plugin-version-new/html.pug deleted file mode 100644 index 86d3d87e8..000000000 --- a/server/lib/emails/plugin-version-new/html.pug +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | New plugin version available | ||
| 5 | |||
| 6 | block content | ||
| 7 | p | ||
| 8 | | A new version of the plugin/theme #{pluginName} is available: #{latestVersion}. | ||
| 9 | | You might want to upgrade it on #[a(href=pluginUrl) the PeerTube admin interface]. | ||
diff --git a/server/lib/emails/user-registered/html.pug b/server/lib/emails/user-registered/html.pug deleted file mode 100644 index 20f62125e..000000000 --- a/server/lib/emails/user-registered/html.pug +++ /dev/null | |||
| @@ -1,10 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | A new user registered | ||
| 5 | |||
| 6 | block content | ||
| 7 | - var mail = user.email || user.pendingEmail; | ||
| 8 | p | ||
| 9 | | User #[a(href=`${WEBSERVER.URL}/accounts/${user.username}`) #{user.username}] just registered. | ||
| 10 | | You might want to contact them at #[a(href=`mailto:${mail}`) #{mail}]. \ No newline at end of file | ||
diff --git a/server/lib/emails/user-registration-request-accepted/html.pug b/server/lib/emails/user-registration-request-accepted/html.pug deleted file mode 100644 index 7a52c3fe1..000000000 --- a/server/lib/emails/user-registration-request-accepted/html.pug +++ /dev/null | |||
| @@ -1,10 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Congratulation #{username}, your registration request has been accepted! | ||
| 5 | |||
| 6 | block content | ||
| 7 | p Your registration request has been accepted. | ||
| 8 | p Moderators sent you the following message: | ||
| 9 | blockquote(style='white-space: pre-wrap') #{moderationResponse} | ||
| 10 | p Your account has been created and you can login on #[a(href=loginLink) #{loginLink}] | ||
diff --git a/server/lib/emails/user-registration-request-rejected/html.pug b/server/lib/emails/user-registration-request-rejected/html.pug deleted file mode 100644 index ec0aa8dfe..000000000 --- a/server/lib/emails/user-registration-request-rejected/html.pug +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Registration request of your account #{username} has rejected | ||
| 5 | |||
| 6 | block content | ||
| 7 | p Your registration request has been rejected. | ||
| 8 | p Moderators sent you the following message: | ||
| 9 | blockquote(style='white-space: pre-wrap') #{moderationResponse} | ||
diff --git a/server/lib/emails/user-registration-request/html.pug b/server/lib/emails/user-registration-request/html.pug deleted file mode 100644 index 64898f3f2..000000000 --- a/server/lib/emails/user-registration-request/html.pug +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | A new user wants to register | ||
| 5 | |||
| 6 | block content | ||
| 7 | p User #{registration.username} wants to register on your PeerTube instance with the following reason: | ||
| 8 | blockquote(style='white-space: pre-wrap') #{registration.registrationReason} | ||
| 9 | p You can accept or reject the registration request in the #[a(href=`${WEBSERVER.URL}/admin/moderation/registrations/list`) administration]. | ||
diff --git a/server/lib/emails/verify-email/html.pug b/server/lib/emails/verify-email/html.pug deleted file mode 100644 index 19ef65f75..000000000 --- a/server/lib/emails/verify-email/html.pug +++ /dev/null | |||
| @@ -1,19 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Email verification | ||
| 5 | |||
| 6 | block content | ||
| 7 | if isRegistrationRequest | ||
| 8 | p You just requested an account on #[a(href=WEBSERVER.URL) #{instanceName}]. | ||
| 9 | else | ||
| 10 | p You just created an account on #[a(href=WEBSERVER.URL) #{instanceName}]. | ||
| 11 | |||
| 12 | if isRegistrationRequest | ||
| 13 | p To complete your registration request you must verify your email first! | ||
| 14 | else | ||
| 15 | p To start using your account you must verify your email first! | ||
| 16 | |||
| 17 | p Please follow #[a(href=verifyEmailUrl) this link] to verify this email belongs to you. | ||
| 18 | p If you can't see the verification link above you can use the following link #[a(href=verifyEmailUrl) #{verifyEmailUrl}] | ||
| 19 | p If you are not the person who initiated this request, please ignore this email. | ||
diff --git a/server/lib/emails/video-abuse-new/html.pug b/server/lib/emails/video-abuse-new/html.pug deleted file mode 100644 index a085b4b38..000000000 --- a/server/lib/emails/video-abuse-new/html.pug +++ /dev/null | |||
| @@ -1,18 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | include ../common/mixins.pug | ||
| 3 | |||
| 4 | block title | ||
| 5 | | A video is pending moderation | ||
| 6 | |||
| 7 | block content | ||
| 8 | p | ||
| 9 | | #[a(href=WEBSERVER.URL) #{instanceName}] received an abuse report for the #{isLocal ? '' : 'remote '}video " | ||
| 10 | a(href=videoUrl) #{videoName} | ||
| 11 | | " by #[+channel(videoChannel)] | ||
| 12 | if videoPublishedAt | ||
| 13 | | , published the #{videoPublishedAt}. | ||
| 14 | else | ||
| 15 | | , uploaded the #{videoCreatedAt} but not yet published. | ||
| 16 | p The reporter, #{reporter}, cited the following reason(s): | ||
| 17 | blockquote #{reason} | ||
| 18 | br(style="display: none;") | ||
diff --git a/server/lib/emails/video-auto-blacklist-new/html.pug b/server/lib/emails/video-auto-blacklist-new/html.pug deleted file mode 100644 index 07c8dfd16..000000000 --- a/server/lib/emails/video-auto-blacklist-new/html.pug +++ /dev/null | |||
| @@ -1,17 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | include ../common/mixins | ||
| 3 | |||
| 4 | block title | ||
| 5 | | A video is pending moderation | ||
| 6 | |||
| 7 | block content | ||
| 8 | p | ||
| 9 | | A recently added video was auto-blacklisted and requires moderator review before going public: | ||
| 10 | | | ||
| 11 | a(href=videoUrl) #{videoName} | ||
| 12 | | | ||
| 13 | | by #[+channel(channel)]. | ||
| 14 | p. | ||
| 15 | Apart from the publisher and the moderation team, no one will be able to see the video until you | ||
| 16 | unblacklist it. If you trust the publisher, any admin can whitelist the user for later videos so | ||
| 17 | that they don't require approval before going public. | ||
diff --git a/server/lib/emails/video-comment-abuse-new/html.pug b/server/lib/emails/video-comment-abuse-new/html.pug deleted file mode 100644 index 752bf7c10..000000000 --- a/server/lib/emails/video-comment-abuse-new/html.pug +++ /dev/null | |||
| @@ -1,16 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | include ../common/mixins.pug | ||
| 3 | |||
| 4 | block title | ||
| 5 | | A comment is pending moderation | ||
| 6 | |||
| 7 | block content | ||
| 8 | p | ||
| 9 | | #[a(href=WEBSERVER.URL) #{instanceName}] received an abuse report for the #{isLocal ? '' : 'remote '} | ||
| 10 | a(href=commentUrl) comment on video "#{videoName}" | ||
| 11 | | of #{flaggedAccount} | ||
| 12 | | created on #{commentCreatedAt} | ||
| 13 | |||
| 14 | p The reporter, #{reporter}, cited the following reason(s): | ||
| 15 | blockquote #{reason} | ||
| 16 | br(style="display: none;") | ||
diff --git a/server/lib/emails/video-comment-mention/html.pug b/server/lib/emails/video-comment-mention/html.pug deleted file mode 100644 index a34c6b090..000000000 --- a/server/lib/emails/video-comment-mention/html.pug +++ /dev/null | |||
| @@ -1,11 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Someone mentioned you | ||
| 5 | |||
| 6 | block content | ||
| 7 | p. | ||
| 8 | #[a(href=accountUrl title=handle) #{accountName}] mentioned you in a comment on video | ||
| 9 | "#[a(href=videoUrl) #{video.name}]": | ||
| 10 | blockquote !{commentHtml} | ||
| 11 | br(style="display: none;") | ||
diff --git a/server/lib/emails/video-comment-new/html.pug b/server/lib/emails/video-comment-new/html.pug deleted file mode 100644 index cbb683fee..000000000 --- a/server/lib/emails/video-comment-new/html.pug +++ /dev/null | |||
| @@ -1,11 +0,0 @@ | |||
| 1 | extends ../common/greetings | ||
| 2 | |||
| 3 | block title | ||
| 4 | | Someone commented your video | ||
| 5 | |||
| 6 | block content | ||
| 7 | p. | ||
| 8 | #[a(href=accountUrl title=handle) #{accountName}] added a comment on your video | ||
| 9 | "#[a(href=videoUrl) #{video.name}]": | ||
| 10 | blockquote !{commentHtml} | ||
| 11 | br(style="display: none;") | ||
diff --git a/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/lib/files-cache/avatar-permanent-file-cache.ts deleted file mode 100644 index 0c508b063..000000000 --- a/server/lib/files-cache/avatar-permanent-file-cache.ts +++ /dev/null | |||
| @@ -1,27 +0,0 @@ | |||
| 1 | import { CONFIG } from '@server/initializers/config' | ||
| 2 | import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' | ||
| 3 | import { ActorImageModel } from '@server/models/actor/actor-image' | ||
| 4 | import { MActorImage } from '@server/types/models' | ||
| 5 | import { AbstractPermanentFileCache } from './shared' | ||
| 6 | |||
| 7 | export class AvatarPermanentFileCache extends AbstractPermanentFileCache<MActorImage> { | ||
| 8 | |||
| 9 | constructor () { | ||
| 10 | super(CONFIG.STORAGE.ACTOR_IMAGES_DIR) | ||
| 11 | } | ||
| 12 | |||
| 13 | protected loadModel (filename: string) { | ||
| 14 | return ActorImageModel.loadByName(filename) | ||
| 15 | } | ||
| 16 | |||
| 17 | protected getImageSize (image: MActorImage): { width: number, height: number } { | ||
| 18 | if (image.width && image.height) { | ||
| 19 | return { | ||
| 20 | height: image.height, | ||
| 21 | width: image.width | ||
| 22 | } | ||
| 23 | } | ||
| 24 | |||
| 25 | return ACTOR_IMAGES_SIZE[image.type][0] | ||
| 26 | } | ||
| 27 | } | ||
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts deleted file mode 100644 index 5630a9b80..000000000 --- a/server/lib/files-cache/index.ts +++ /dev/null | |||
| @@ -1,6 +0,0 @@ | |||
| 1 | export * from './avatar-permanent-file-cache' | ||
| 2 | export * from './video-miniature-permanent-file-cache' | ||
| 3 | export * from './video-captions-simple-file-cache' | ||
| 4 | export * from './video-previews-simple-file-cache' | ||
| 5 | export * from './video-storyboards-simple-file-cache' | ||
| 6 | export * from './video-torrents-simple-file-cache' | ||
diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts deleted file mode 100644 index f990e9872..000000000 --- a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts +++ /dev/null | |||
| @@ -1,132 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { LRUCache } from 'lru-cache' | ||
| 3 | import { Model } from 'sequelize' | ||
| 4 | import { logger } from '@server/helpers/logger' | ||
| 5 | import { CachePromise } from '@server/helpers/promise-cache' | ||
| 6 | import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants' | ||
| 7 | import { downloadImageFromWorker } from '@server/lib/worker/parent-process' | ||
| 8 | import { HttpStatusCode } from '@shared/models' | ||
| 9 | |||
| 10 | type ImageModel = { | ||
| 11 | fileUrl: string | ||
| 12 | filename: string | ||
| 13 | onDisk: boolean | ||
| 14 | |||
| 15 | isOwned (): boolean | ||
| 16 | getPath (): string | ||
| 17 | |||
| 18 | save (): Promise<Model> | ||
| 19 | } | ||
| 20 | |||
| 21 | export abstract class AbstractPermanentFileCache <M extends ImageModel> { | ||
| 22 | // Unsafe because it can return paths that do not exist anymore | ||
| 23 | private readonly filenameToPathUnsafeCache = new LRUCache<string, string>({ | ||
| 24 | max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE | ||
| 25 | }) | ||
| 26 | |||
| 27 | protected abstract getImageSize (image: M): { width: number, height: number } | ||
| 28 | protected abstract loadModel (filename: string): Promise<M> | ||
| 29 | |||
| 30 | constructor (private readonly directory: string) { | ||
| 31 | |||
| 32 | } | ||
| 33 | |||
| 34 | async lazyServe (options: { | ||
| 35 | filename: string | ||
| 36 | res: express.Response | ||
| 37 | next: express.NextFunction | ||
| 38 | }) { | ||
| 39 | const { filename, res, next } = options | ||
| 40 | |||
| 41 | if (this.filenameToPathUnsafeCache.has(filename)) { | ||
| 42 | return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) | ||
| 43 | } | ||
| 44 | |||
| 45 | const image = await this.lazyLoadIfNeeded(filename) | ||
| 46 | if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end() | ||
| 47 | |||
| 48 | const path = image.getPath() | ||
| 49 | this.filenameToPathUnsafeCache.set(filename, path) | ||
| 50 | |||
| 51 | return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => { | ||
| 52 | if (!err) return | ||
| 53 | |||
| 54 | this.onServeError({ err, image, next, filename }) | ||
| 55 | }) | ||
| 56 | } | ||
| 57 | |||
| 58 | @CachePromise({ | ||
| 59 | keyBuilder: filename => filename | ||
| 60 | }) | ||
| 61 | private async lazyLoadIfNeeded (filename: string) { | ||
| 62 | const image = await this.loadModel(filename) | ||
| 63 | if (!image) return undefined | ||
| 64 | |||
| 65 | if (image.onDisk === false) { | ||
| 66 | if (!image.fileUrl) return undefined | ||
| 67 | |||
| 68 | try { | ||
| 69 | await this.downloadRemoteFile(image) | ||
| 70 | } catch (err) { | ||
| 71 | logger.warn('Cannot process remote image %s.', image.fileUrl, { err }) | ||
| 72 | |||
| 73 | return undefined | ||
| 74 | } | ||
| 75 | } | ||
| 76 | |||
| 77 | return image | ||
| 78 | } | ||
| 79 | |||
| 80 | async downloadRemoteFile (image: M) { | ||
| 81 | logger.info('Download remote image %s lazily.', image.fileUrl) | ||
| 82 | |||
| 83 | const destination = await this.downloadImage({ | ||
| 84 | filename: image.filename, | ||
| 85 | fileUrl: image.fileUrl, | ||
| 86 | size: this.getImageSize(image) | ||
| 87 | }) | ||
| 88 | |||
| 89 | image.onDisk = true | ||
| 90 | image.save() | ||
| 91 | .catch(err => logger.error('Cannot save new image disk state.', { err })) | ||
| 92 | |||
| 93 | return destination | ||
| 94 | } | ||
| 95 | |||
| 96 | private onServeError (options: { | ||
| 97 | err: any | ||
| 98 | image: M | ||
| 99 | filename: string | ||
| 100 | next: express.NextFunction | ||
| 101 | }) { | ||
| 102 | const { err, image, filename, next } = options | ||
| 103 | |||
| 104 | // It seems this actor image is not on the disk anymore | ||
| 105 | if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) { | ||
| 106 | logger.error('Cannot lazy serve image %s.', filename, { err }) | ||
| 107 | |||
| 108 | this.filenameToPathUnsafeCache.delete(filename) | ||
| 109 | |||
| 110 | image.onDisk = false | ||
| 111 | image.save() | ||
| 112 | .catch(err => logger.error('Cannot save new image disk state.', { err })) | ||
| 113 | } | ||
| 114 | |||
| 115 | return next(err) | ||
| 116 | } | ||
| 117 | |||
| 118 | private downloadImage (options: { | ||
| 119 | fileUrl: string | ||
| 120 | filename: string | ||
| 121 | size: { width: number, height: number } | ||
| 122 | }) { | ||
| 123 | const downloaderOptions = { | ||
| 124 | url: options.fileUrl, | ||
| 125 | destDir: this.directory, | ||
| 126 | destName: options.filename, | ||
| 127 | size: options.size | ||
| 128 | } | ||
| 129 | |||
| 130 | return downloadImageFromWorker(downloaderOptions) | ||
| 131 | } | ||
| 132 | } | ||
diff --git a/server/lib/files-cache/shared/abstract-simple-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts deleted file mode 100644 index 6fab322cd..000000000 --- a/server/lib/files-cache/shared/abstract-simple-file-cache.ts +++ /dev/null | |||
| @@ -1,30 +0,0 @@ | |||
| 1 | import { remove } from 'fs-extra' | ||
| 2 | import { logger } from '../../../helpers/logger' | ||
| 3 | import memoizee from 'memoizee' | ||
| 4 | |||
| 5 | type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined | ||
| 6 | |||
| 7 | export abstract class AbstractSimpleFileCache <T> { | ||
| 8 | |||
| 9 | getFilePath: (params: T) => Promise<GetFilePathResult> | ||
| 10 | |||
| 11 | abstract getFilePathImpl (params: T): Promise<GetFilePathResult> | ||
| 12 | |||
| 13 | // Load and save the remote file, then return the local path from filesystem | ||
| 14 | protected abstract loadRemoteFile (key: string): Promise<GetFilePathResult> | ||
| 15 | |||
| 16 | init (max: number, maxAge: number) { | ||
| 17 | this.getFilePath = memoizee(this.getFilePathImpl, { | ||
| 18 | maxAge, | ||
| 19 | max, | ||
| 20 | promise: true, | ||
| 21 | dispose: (result?: GetFilePathResult) => { | ||
| 22 | if (result && result.isOwned !== true) { | ||
| 23 | remove(result.path) | ||
| 24 | .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name)) | ||
| 25 | .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err })) | ||
| 26 | } | ||
| 27 | } | ||
| 28 | }) | ||
| 29 | } | ||
| 30 | } | ||
diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts deleted file mode 100644 index 61c4aacc7..000000000 --- a/server/lib/files-cache/shared/index.ts +++ /dev/null | |||
| @@ -1,2 +0,0 @@ | |||
| 1 | export * from './abstract-permanent-file-cache' | ||
| 2 | export * from './abstract-simple-file-cache' | ||
diff --git a/server/lib/files-cache/video-captions-simple-file-cache.ts b/server/lib/files-cache/video-captions-simple-file-cache.ts deleted file mode 100644 index cbeeff732..000000000 --- a/server/lib/files-cache/video-captions-simple-file-cache.ts +++ /dev/null | |||
| @@ -1,61 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | ||
| 4 | import { CONFIG } from '../../initializers/config' | ||
| 5 | import { FILES_CACHE } from '../../initializers/constants' | ||
| 6 | import { VideoModel } from '../../models/video/video' | ||
| 7 | import { VideoCaptionModel } from '../../models/video/video-caption' | ||
| 8 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' | ||
| 9 | |||
| 10 | class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache <string> { | ||
| 11 | |||
| 12 | private static instance: VideoCaptionsSimpleFileCache | ||
| 13 | |||
| 14 | private constructor () { | ||
| 15 | super() | ||
| 16 | } | ||
| 17 | |||
| 18 | static get Instance () { | ||
| 19 | return this.instance || (this.instance = new this()) | ||
| 20 | } | ||
| 21 | |||
| 22 | async getFilePathImpl (filename: string) { | ||
| 23 | const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename) | ||
| 24 | if (!videoCaption) return undefined | ||
| 25 | |||
| 26 | if (videoCaption.isOwned()) { | ||
| 27 | return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } | ||
| 28 | } | ||
| 29 | |||
| 30 | return this.loadRemoteFile(filename) | ||
| 31 | } | ||
| 32 | |||
| 33 | // Key is the caption filename | ||
| 34 | protected async loadRemoteFile (key: string) { | ||
| 35 | const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(key) | ||
| 36 | if (!videoCaption) return undefined | ||
| 37 | |||
| 38 | if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') | ||
| 39 | |||
| 40 | // Used to fetch the path | ||
| 41 | const video = await VideoModel.loadFull(videoCaption.videoId) | ||
| 42 | if (!video) return undefined | ||
| 43 | |||
| 44 | const remoteUrl = videoCaption.getFileUrl(video) | ||
| 45 | const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) | ||
| 46 | |||
| 47 | try { | ||
| 48 | await doRequestAndSaveToFile(remoteUrl, destPath) | ||
| 49 | |||
| 50 | return { isOwned: false, path: destPath } | ||
| 51 | } catch (err) { | ||
| 52 | logger.info('Cannot fetch remote caption file %s.', remoteUrl, { err }) | ||
| 53 | |||
| 54 | return undefined | ||
| 55 | } | ||
| 56 | } | ||
| 57 | } | ||
| 58 | |||
| 59 | export { | ||
| 60 | VideoCaptionsSimpleFileCache | ||
| 61 | } | ||
diff --git a/server/lib/files-cache/video-miniature-permanent-file-cache.ts b/server/lib/files-cache/video-miniature-permanent-file-cache.ts deleted file mode 100644 index 35d9466f7..000000000 --- a/server/lib/files-cache/video-miniature-permanent-file-cache.ts +++ /dev/null | |||
| @@ -1,28 +0,0 @@ | |||
| 1 | import { CONFIG } from '@server/initializers/config' | ||
| 2 | import { THUMBNAILS_SIZE } from '@server/initializers/constants' | ||
| 3 | import { ThumbnailModel } from '@server/models/video/thumbnail' | ||
| 4 | import { MThumbnail } from '@server/types/models' | ||
| 5 | import { ThumbnailType } from '@shared/models' | ||
| 6 | import { AbstractPermanentFileCache } from './shared' | ||
| 7 | |||
| 8 | export class VideoMiniaturePermanentFileCache extends AbstractPermanentFileCache<MThumbnail> { | ||
| 9 | |||
| 10 | constructor () { | ||
| 11 | super(CONFIG.STORAGE.THUMBNAILS_DIR) | ||
| 12 | } | ||
| 13 | |||
| 14 | protected loadModel (filename: string) { | ||
| 15 | return ThumbnailModel.loadByFilename(filename, ThumbnailType.MINIATURE) | ||
| 16 | } | ||
| 17 | |||
| 18 | protected getImageSize (image: MThumbnail): { width: number, height: number } { | ||
| 19 | if (image.width && image.height) { | ||
| 20 | return { | ||
| 21 | height: image.height, | ||
| 22 | width: image.width | ||
| 23 | } | ||
| 24 | } | ||
| 25 | |||
| 26 | return THUMBNAILS_SIZE | ||
| 27 | } | ||
| 28 | } | ||
diff --git a/server/lib/files-cache/video-previews-simple-file-cache.ts b/server/lib/files-cache/video-previews-simple-file-cache.ts deleted file mode 100644 index a05e80e16..000000000 --- a/server/lib/files-cache/video-previews-simple-file-cache.ts +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import { FILES_CACHE } from '../../initializers/constants' | ||
| 3 | import { VideoModel } from '../../models/video/video' | ||
| 4 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' | ||
| 5 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | ||
| 6 | import { ThumbnailModel } from '@server/models/video/thumbnail' | ||
| 7 | import { ThumbnailType } from '@shared/models' | ||
| 8 | import { logger } from '@server/helpers/logger' | ||
| 9 | |||
| 10 | class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache <string> { | ||
| 11 | |||
| 12 | private static instance: VideoPreviewsSimpleFileCache | ||
| 13 | |||
| 14 | private constructor () { | ||
| 15 | super() | ||
| 16 | } | ||
| 17 | |||
| 18 | static get Instance () { | ||
| 19 | return this.instance || (this.instance = new this()) | ||
| 20 | } | ||
| 21 | |||
| 22 | async getFilePathImpl (filename: string) { | ||
| 23 | const thumbnail = await ThumbnailModel.loadWithVideoByFilename(filename, ThumbnailType.PREVIEW) | ||
| 24 | if (!thumbnail) return undefined | ||
| 25 | |||
| 26 | if (thumbnail.Video.isOwned()) return { isOwned: true, path: thumbnail.getPath() } | ||
| 27 | |||
| 28 | return this.loadRemoteFile(thumbnail.Video.uuid) | ||
| 29 | } | ||
| 30 | |||
| 31 | // Key is the video UUID | ||
| 32 | protected async loadRemoteFile (key: string) { | ||
| 33 | const video = await VideoModel.loadFull(key) | ||
| 34 | if (!video) return undefined | ||
| 35 | |||
| 36 | if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') | ||
| 37 | |||
| 38 | const preview = video.getPreview() | ||
| 39 | const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename) | ||
| 40 | const remoteUrl = preview.getOriginFileUrl(video) | ||
| 41 | |||
| 42 | try { | ||
| 43 | await doRequestAndSaveToFile(remoteUrl, destPath) | ||
| 44 | |||
| 45 | logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath) | ||
| 46 | |||
| 47 | return { isOwned: false, path: destPath } | ||
| 48 | } catch (err) { | ||
| 49 | logger.info('Cannot fetch remote preview file %s.', remoteUrl, { err }) | ||
| 50 | |||
| 51 | return undefined | ||
| 52 | } | ||
| 53 | } | ||
| 54 | } | ||
| 55 | |||
| 56 | export { | ||
| 57 | VideoPreviewsSimpleFileCache | ||
| 58 | } | ||
diff --git a/server/lib/files-cache/video-storyboards-simple-file-cache.ts b/server/lib/files-cache/video-storyboards-simple-file-cache.ts deleted file mode 100644 index 4cd96e70c..000000000 --- a/server/lib/files-cache/video-storyboards-simple-file-cache.ts +++ /dev/null | |||
| @@ -1,53 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | ||
| 4 | import { StoryboardModel } from '@server/models/video/storyboard' | ||
| 5 | import { FILES_CACHE } from '../../initializers/constants' | ||
| 6 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' | ||
| 7 | |||
| 8 | class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache <string> { | ||
| 9 | |||
| 10 | private static instance: VideoStoryboardsSimpleFileCache | ||
| 11 | |||
| 12 | private constructor () { | ||
| 13 | super() | ||
| 14 | } | ||
| 15 | |||
| 16 | static get Instance () { | ||
| 17 | return this.instance || (this.instance = new this()) | ||
| 18 | } | ||
| 19 | |||
| 20 | async getFilePathImpl (filename: string) { | ||
| 21 | const storyboard = await StoryboardModel.loadWithVideoByFilename(filename) | ||
| 22 | if (!storyboard) return undefined | ||
| 23 | |||
| 24 | if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() } | ||
| 25 | |||
| 26 | return this.loadRemoteFile(storyboard.filename) | ||
| 27 | } | ||
| 28 | |||
| 29 | // Key is the storyboard filename | ||
| 30 | protected async loadRemoteFile (key: string) { | ||
| 31 | const storyboard = await StoryboardModel.loadWithVideoByFilename(key) | ||
| 32 | if (!storyboard) return undefined | ||
| 33 | |||
| 34 | const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename) | ||
| 35 | const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video) | ||
| 36 | |||
| 37 | try { | ||
| 38 | await doRequestAndSaveToFile(remoteUrl, destPath) | ||
| 39 | |||
| 40 | logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath) | ||
| 41 | |||
| 42 | return { isOwned: false, path: destPath } | ||
| 43 | } catch (err) { | ||
| 44 | logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err }) | ||
| 45 | |||
| 46 | return undefined | ||
| 47 | } | ||
| 48 | } | ||
| 49 | } | ||
| 50 | |||
| 51 | export { | ||
| 52 | VideoStoryboardsSimpleFileCache | ||
| 53 | } | ||
diff --git a/server/lib/files-cache/video-torrents-simple-file-cache.ts b/server/lib/files-cache/video-torrents-simple-file-cache.ts deleted file mode 100644 index 8bcd0b9bf..000000000 --- a/server/lib/files-cache/video-torrents-simple-file-cache.ts +++ /dev/null | |||
| @@ -1,70 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | ||
| 4 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 5 | import { MVideo, MVideoFile } from '@server/types/models' | ||
| 6 | import { CONFIG } from '../../initializers/config' | ||
| 7 | import { FILES_CACHE } from '../../initializers/constants' | ||
| 8 | import { VideoModel } from '../../models/video/video' | ||
| 9 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' | ||
| 10 | |||
| 11 | class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache <string> { | ||
| 12 | |||
| 13 | private static instance: VideoTorrentsSimpleFileCache | ||
| 14 | |||
| 15 | private constructor () { | ||
| 16 | super() | ||
| 17 | } | ||
| 18 | |||
| 19 | static get Instance () { | ||
| 20 | return this.instance || (this.instance = new this()) | ||
| 21 | } | ||
| 22 | |||
| 23 | async getFilePathImpl (filename: string) { | ||
| 24 | const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(filename) | ||
| 25 | if (!file) return undefined | ||
| 26 | |||
| 27 | if (file.getVideo().isOwned()) { | ||
| 28 | const downloadName = this.buildDownloadName(file.getVideo(), file) | ||
| 29 | |||
| 30 | return { isOwned: true, path: join(CONFIG.STORAGE.TORRENTS_DIR, file.torrentFilename), downloadName } | ||
| 31 | } | ||
| 32 | |||
| 33 | return this.loadRemoteFile(filename) | ||
| 34 | } | ||
| 35 | |||
| 36 | // Key is the torrent filename | ||
| 37 | protected async loadRemoteFile (key: string) { | ||
| 38 | const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(key) | ||
| 39 | if (!file) return undefined | ||
| 40 | |||
| 41 | if (file.getVideo().isOwned()) throw new Error('Cannot load remote file of owned video.') | ||
| 42 | |||
| 43 | // Used to fetch the path | ||
| 44 | const video = await VideoModel.loadFull(file.getVideo().id) | ||
| 45 | if (!video) return undefined | ||
| 46 | |||
| 47 | const remoteUrl = file.getRemoteTorrentUrl(video) | ||
| 48 | const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) | ||
| 49 | |||
| 50 | try { | ||
| 51 | await doRequestAndSaveToFile(remoteUrl, destPath) | ||
| 52 | |||
| 53 | const downloadName = this.buildDownloadName(video, file) | ||
| 54 | |||
| 55 | return { isOwned: false, path: destPath, downloadName } | ||
| 56 | } catch (err) { | ||
| 57 | logger.info('Cannot fetch remote torrent file %s.', remoteUrl, { err }) | ||
| 58 | |||
| 59 | return undefined | ||
| 60 | } | ||
| 61 | } | ||
| 62 | |||
| 63 | private buildDownloadName (video: MVideo, file: MVideoFile) { | ||
| 64 | return `${video.name}-${file.resolution}p.torrent` | ||
| 65 | } | ||
| 66 | } | ||
| 67 | |||
| 68 | export { | ||
| 69 | VideoTorrentsSimpleFileCache | ||
| 70 | } | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts deleted file mode 100644 index 19044d7c2..000000000 --- a/server/lib/hls.ts +++ /dev/null | |||
| @@ -1,285 +0,0 @@ | |||
| 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' | ||
| 2 | import { flatten } from 'lodash' | ||
| 3 | import PQueue from 'p-queue' | ||
| 4 | import { basename, dirname, join } from 'path' | ||
| 5 | import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' | ||
| 6 | import { uniqify, uuidRegex } from '@shared/core-utils' | ||
| 7 | import { sha256 } from '@shared/extra-utils' | ||
| 8 | import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' | ||
| 9 | import { VideoStorage } from '@shared/models' | ||
| 10 | import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' | ||
| 11 | import { logger, loggerTagsFactory } from '../helpers/logger' | ||
| 12 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | ||
| 13 | import { generateRandomString } from '../helpers/utils' | ||
| 14 | import { CONFIG } from '../initializers/config' | ||
| 15 | import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' | ||
| 16 | import { sequelizeTypescript } from '../initializers/database' | ||
| 17 | import { VideoFileModel } from '../models/video/video-file' | ||
| 18 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
| 19 | import { storeHLSFileFromFilename } from './object-storage' | ||
| 20 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' | ||
| 21 | import { VideoPathManager } from './video-path-manager' | ||
| 22 | |||
| 23 | const lTags = loggerTagsFactory('hls') | ||
| 24 | |||
| 25 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | ||
| 26 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | ||
| 27 | |||
| 28 | // Use separate SQL queries, because we could have many videos to update | ||
| 29 | for (const playlist of playlistsToUpdate) { | ||
| 30 | await sequelizeTypescript.transaction(async t => { | ||
| 31 | const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) | ||
| 32 | |||
| 33 | playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) | ||
| 34 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION | ||
| 35 | |||
| 36 | await playlist.save({ transaction: t }) | ||
| 37 | }) | ||
| 38 | } | ||
| 39 | } | ||
| 40 | |||
| 41 | async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { | ||
| 42 | try { | ||
| 43 | let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) | ||
| 44 | playlistWithFiles = await updateSha256VODSegments(video, playlist) | ||
| 45 | |||
| 46 | // Refresh playlist, operations can take some time | ||
| 47 | playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) | ||
| 48 | playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) | ||
| 49 | await playlistWithFiles.save() | ||
| 50 | |||
| 51 | video.setHLSPlaylist(playlistWithFiles) | ||
| 52 | } catch (err) { | ||
| 53 | logger.warn('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) | ||
| 54 | } | ||
| 55 | } | ||
| 56 | |||
| 57 | // --------------------------------------------------------------------------- | ||
| 58 | |||
| 59 | // Avoid concurrency issues when updating streaming playlist files | ||
| 60 | const playlistFilesQueue = new PQueue({ concurrency: 1 }) | ||
| 61 | |||
| 62 | function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> { | ||
| 63 | return playlistFilesQueue.add(async () => { | ||
| 64 | const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) | ||
| 65 | |||
| 66 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | ||
| 67 | |||
| 68 | for (const file of playlist.VideoFiles) { | ||
| 69 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | ||
| 70 | |||
| 71 | await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { | ||
| 72 | const size = await getVideoStreamDimensionsInfo(videoFilePath) | ||
| 73 | |||
| 74 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | ||
| 75 | const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` | ||
| 76 | |||
| 77 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | ||
| 78 | if (file.fps) line += ',FRAME-RATE=' + file.fps | ||
| 79 | |||
| 80 | const codecs = await Promise.all([ | ||
| 81 | getVideoStreamCodec(videoFilePath), | ||
| 82 | getAudioStreamCodec(videoFilePath) | ||
| 83 | ]) | ||
| 84 | |||
| 85 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` | ||
| 86 | |||
| 87 | masterPlaylists.push(line) | ||
| 88 | masterPlaylists.push(playlistFilename) | ||
| 89 | }) | ||
| 90 | } | ||
| 91 | |||
| 92 | if (playlist.playlistFilename) { | ||
| 93 | await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) | ||
| 94 | } | ||
| 95 | playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) | ||
| 96 | |||
| 97 | const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) | ||
| 98 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
| 99 | |||
| 100 | logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid)) | ||
| 101 | |||
| 102 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | ||
| 103 | playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) | ||
| 104 | await remove(masterPlaylistPath) | ||
| 105 | } | ||
| 106 | |||
| 107 | return playlist.save() | ||
| 108 | }) | ||
| 109 | } | ||
| 110 | |||
| 111 | // --------------------------------------------------------------------------- | ||
| 112 | |||
| 113 | function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> { | ||
| 114 | return playlistFilesQueue.add(async () => { | ||
| 115 | const json: { [filename: string]: { [range: string]: string } } = {} | ||
| 116 | |||
| 117 | const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) | ||
| 118 | |||
| 119 | // For all the resolutions available for this video | ||
| 120 | for (const file of playlist.VideoFiles) { | ||
| 121 | const rangeHashes: { [range: string]: string } = {} | ||
| 122 | const fileWithPlaylist = file.withVideoOrPlaylist(playlist) | ||
| 123 | |||
| 124 | await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { | ||
| 125 | |||
| 126 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { | ||
| 127 | const playlistContent = await readFile(resolutionPlaylistPath) | ||
| 128 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
| 129 | |||
| 130 | const fd = await open(videoPath, 'r') | ||
| 131 | for (const range of ranges) { | ||
| 132 | const buf = Buffer.alloc(range.length) | ||
| 133 | await read(fd, buf, 0, range.length, range.offset) | ||
| 134 | |||
| 135 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | ||
| 136 | } | ||
| 137 | await close(fd) | ||
| 138 | |||
| 139 | const videoFilename = file.filename | ||
| 140 | json[videoFilename] = rangeHashes | ||
| 141 | }) | ||
| 142 | }) | ||
| 143 | } | ||
| 144 | |||
| 145 | if (playlist.segmentsSha256Filename) { | ||
| 146 | await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) | ||
| 147 | } | ||
| 148 | playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) | ||
| 149 | |||
| 150 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) | ||
| 151 | await outputJSON(outputPath, json) | ||
| 152 | |||
| 153 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | ||
| 154 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) | ||
| 155 | await remove(outputPath) | ||
| 156 | } | ||
| 157 | |||
| 158 | return playlist.save() | ||
| 159 | }) | ||
| 160 | } | ||
| 161 | |||
| 162 | // --------------------------------------------------------------------------- | ||
| 163 | |||
| 164 | async function buildSha256Segment (segmentPath: string) { | ||
| 165 | const buf = await readFile(segmentPath) | ||
| 166 | return sha256(buf) | ||
| 167 | } | ||
| 168 | |||
| 169 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { | ||
| 170 | let timer | ||
| 171 | let remainingBodyKBLimit = bodyKBLimit | ||
| 172 | |||
| 173 | logger.info('Importing HLS playlist %s', playlistUrl) | ||
| 174 | |||
| 175 | return new Promise<void>(async (res, rej) => { | ||
| 176 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) | ||
| 177 | |||
| 178 | await ensureDir(tmpDirectory) | ||
| 179 | |||
| 180 | timer = setTimeout(() => { | ||
| 181 | deleteTmpDirectory(tmpDirectory) | ||
| 182 | |||
| 183 | return rej(new Error('HLS download timeout.')) | ||
| 184 | }, timeout) | ||
| 185 | |||
| 186 | try { | ||
| 187 | // Fetch master playlist | ||
| 188 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | ||
| 189 | |||
| 190 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | ||
| 191 | const fileUrls = uniqify(flatten(await Promise.all(subRequests))) | ||
| 192 | |||
| 193 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | ||
| 194 | |||
| 195 | for (const fileUrl of fileUrls) { | ||
| 196 | const destPath = join(tmpDirectory, basename(fileUrl)) | ||
| 197 | |||
| 198 | await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) | ||
| 199 | |||
| 200 | const { size } = await stat(destPath) | ||
| 201 | remainingBodyKBLimit -= (size / 1000) | ||
| 202 | |||
| 203 | logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) | ||
| 204 | } | ||
| 205 | |||
| 206 | clearTimeout(timer) | ||
| 207 | |||
| 208 | await move(tmpDirectory, destinationDir, { overwrite: true }) | ||
| 209 | |||
| 210 | return res() | ||
| 211 | } catch (err) { | ||
| 212 | deleteTmpDirectory(tmpDirectory) | ||
| 213 | |||
| 214 | return rej(err) | ||
| 215 | } | ||
| 216 | }) | ||
| 217 | |||
| 218 | function deleteTmpDirectory (directory: string) { | ||
| 219 | remove(directory) | ||
| 220 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | ||
| 221 | } | ||
| 222 | |||
| 223 | async function fetchUniqUrls (playlistUrl: string) { | ||
| 224 | const { body } = await doRequest(playlistUrl) | ||
| 225 | |||
| 226 | if (!body) return [] | ||
| 227 | |||
| 228 | const urls = body.split('\n') | ||
| 229 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | ||
| 230 | .map(url => { | ||
| 231 | if (url.startsWith('http://') || url.startsWith('https://')) return url | ||
| 232 | |||
| 233 | return `${dirname(playlistUrl)}/${url}` | ||
| 234 | }) | ||
| 235 | |||
| 236 | return uniqify(urls) | ||
| 237 | } | ||
| 238 | } | ||
| 239 | |||
| 240 | // --------------------------------------------------------------------------- | ||
| 241 | |||
| 242 | async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { | ||
| 243 | const content = await readFile(playlistPath, 'utf8') | ||
| 244 | |||
| 245 | const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) | ||
| 246 | |||
| 247 | await writeFile(playlistPath, newContent, 'utf8') | ||
| 248 | } | ||
| 249 | |||
| 250 | // --------------------------------------------------------------------------- | ||
| 251 | |||
| 252 | function injectQueryToPlaylistUrls (content: string, queryString: string) { | ||
| 253 | return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) | ||
| 254 | } | ||
| 255 | |||
| 256 | // --------------------------------------------------------------------------- | ||
| 257 | |||
| 258 | export { | ||
| 259 | updateMasterHLSPlaylist, | ||
| 260 | updateSha256VODSegments, | ||
| 261 | buildSha256Segment, | ||
| 262 | downloadPlaylistSegments, | ||
| 263 | updateStreamingPlaylistsInfohashesIfNeeded, | ||
| 264 | updatePlaylistAfterFileChange, | ||
| 265 | injectQueryToPlaylistUrls, | ||
| 266 | renameVideoFileInPlaylist | ||
| 267 | } | ||
| 268 | |||
| 269 | // --------------------------------------------------------------------------- | ||
| 270 | |||
| 271 | function getRangesFromPlaylist (playlistContent: string) { | ||
| 272 | const ranges: { offset: number, length: number }[] = [] | ||
| 273 | const lines = playlistContent.split('\n') | ||
| 274 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | ||
| 275 | |||
| 276 | for (const line of lines) { | ||
| 277 | const captured = regex.exec(line) | ||
| 278 | |||
| 279 | if (captured) { | ||
| 280 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | ||
| 281 | } | ||
| 282 | } | ||
| 283 | |||
| 284 | return ranges | ||
| 285 | } | ||
diff --git a/server/lib/internal-event-emitter.ts b/server/lib/internal-event-emitter.ts deleted file mode 100644 index 08b46a5c3..000000000 --- a/server/lib/internal-event-emitter.ts +++ /dev/null | |||
| @@ -1,35 +0,0 @@ | |||
| 1 | import { MChannel, MVideo } from '@server/types/models' | ||
| 2 | import { EventEmitter } from 'events' | ||
| 3 | |||
| 4 | export interface PeerTubeInternalEvents { | ||
| 5 | 'video-created': (options: { video: MVideo }) => void | ||
| 6 | 'video-updated': (options: { video: MVideo }) => void | ||
| 7 | 'video-deleted': (options: { video: MVideo }) => void | ||
| 8 | |||
| 9 | 'channel-created': (options: { channel: MChannel }) => void | ||
| 10 | 'channel-updated': (options: { channel: MChannel }) => void | ||
| 11 | 'channel-deleted': (options: { channel: MChannel }) => void | ||
| 12 | } | ||
| 13 | |||
| 14 | declare interface InternalEventEmitter { | ||
| 15 | on<U extends keyof PeerTubeInternalEvents>( | ||
| 16 | event: U, listener: PeerTubeInternalEvents[U] | ||
| 17 | ): this | ||
| 18 | |||
| 19 | emit<U extends keyof PeerTubeInternalEvents>( | ||
| 20 | event: U, ...args: Parameters<PeerTubeInternalEvents[U]> | ||
| 21 | ): boolean | ||
| 22 | } | ||
| 23 | |||
| 24 | class InternalEventEmitter extends EventEmitter { | ||
| 25 | |||
| 26 | private static instance: InternalEventEmitter | ||
| 27 | |||
| 28 | static get Instance () { | ||
| 29 | return this.instance || (this.instance = new this()) | ||
| 30 | } | ||
| 31 | } | ||
| 32 | |||
| 33 | export { | ||
| 34 | InternalEventEmitter | ||
| 35 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts deleted file mode 100644 index 6ee9e2429..000000000 --- a/server/lib/job-queue/handlers/activitypub-cleaner.ts +++ /dev/null | |||
| @@ -1,202 +0,0 @@ | |||
| 1 | import { map } from 'bluebird' | ||
| 2 | import { Job } from 'bullmq' | ||
| 3 | import { | ||
| 4 | isAnnounceActivityValid, | ||
| 5 | isDislikeActivityValid, | ||
| 6 | isLikeActivityValid | ||
| 7 | } from '@server/helpers/custom-validators/activitypub/activity' | ||
| 8 | import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' | ||
| 9 | import { PeerTubeRequestError } from '@server/helpers/requests' | ||
| 10 | import { AP_CLEANER } from '@server/initializers/constants' | ||
| 11 | import { fetchAP } from '@server/lib/activitypub/activity' | ||
| 12 | import { checkUrlsSameHost } from '@server/lib/activitypub/url' | ||
| 13 | import { Redis } from '@server/lib/redis' | ||
| 14 | import { VideoModel } from '@server/models/video/video' | ||
| 15 | import { VideoCommentModel } from '@server/models/video/video-comment' | ||
| 16 | import { VideoShareModel } from '@server/models/video/video-share' | ||
| 17 | import { HttpStatusCode } from '@shared/models' | ||
| 18 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | ||
| 19 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
| 20 | |||
| 21 | const lTags = loggerTagsFactory('ap-cleaner') | ||
| 22 | |||
| 23 | // Job to clean remote interactions off local videos | ||
| 24 | |||
| 25 | async function processActivityPubCleaner (_job: Job) { | ||
| 26 | logger.info('Processing ActivityPub cleaner.', lTags()) | ||
| 27 | |||
| 28 | { | ||
| 29 | const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() | ||
| 30 | const { bodyValidator, deleter, updater } = rateOptionsFactory() | ||
| 31 | |||
| 32 | await map(rateUrls, async rateUrl => { | ||
| 33 | // TODO: remove when https://github.com/mastodon/mastodon/issues/13571 is fixed | ||
| 34 | if (rateUrl.includes('#')) return | ||
| 35 | |||
| 36 | const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter }) | ||
| 37 | |||
| 38 | if (result?.status === 'deleted') { | ||
| 39 | const { videoId, type } = result.data | ||
| 40 | |||
| 41 | await VideoModel.syncLocalRates(videoId, type, undefined) | ||
| 42 | } | ||
| 43 | }, { concurrency: AP_CLEANER.CONCURRENCY }) | ||
| 44 | } | ||
| 45 | |||
| 46 | { | ||
| 47 | const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos() | ||
| 48 | const { bodyValidator, deleter, updater } = shareOptionsFactory() | ||
| 49 | |||
| 50 | await map(shareUrls, async shareUrl => { | ||
| 51 | await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter }) | ||
| 52 | }, { concurrency: AP_CLEANER.CONCURRENCY }) | ||
| 53 | } | ||
| 54 | |||
| 55 | { | ||
| 56 | const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos() | ||
| 57 | const { bodyValidator, deleter, updater } = commentOptionsFactory() | ||
| 58 | |||
| 59 | await map(commentUrls, async commentUrl => { | ||
| 60 | await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter }) | ||
| 61 | }, { concurrency: AP_CLEANER.CONCURRENCY }) | ||
| 62 | } | ||
| 63 | } | ||
| 64 | |||
| 65 | // --------------------------------------------------------------------------- | ||
| 66 | |||
| 67 | export { | ||
| 68 | processActivityPubCleaner | ||
| 69 | } | ||
| 70 | |||
| 71 | // --------------------------------------------------------------------------- | ||
| 72 | |||
| 73 | async function updateObjectIfNeeded <T> (options: { | ||
| 74 | url: string | ||
| 75 | bodyValidator: (body: any) => boolean | ||
| 76 | updater: (url: string, newUrl: string) => Promise<T> | ||
| 77 | deleter: (url: string) => Promise<T> } | ||
| 78 | ): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { | ||
| 79 | const { url, bodyValidator, updater, deleter } = options | ||
| 80 | |||
| 81 | const on404OrTombstone = async () => { | ||
| 82 | logger.info('Removing remote AP object %s.', url, lTags(url)) | ||
| 83 | const data = await deleter(url) | ||
| 84 | |||
| 85 | return { status: 'deleted' as 'deleted', data } | ||
| 86 | } | ||
| 87 | |||
| 88 | try { | ||
| 89 | const { body } = await fetchAP<any>(url) | ||
| 90 | |||
| 91 | // If not same id, check same host and update | ||
| 92 | if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) | ||
| 93 | |||
| 94 | if (body.type === 'Tombstone') { | ||
| 95 | return on404OrTombstone() | ||
| 96 | } | ||
| 97 | |||
| 98 | const newUrl = body.id | ||
| 99 | if (newUrl !== url) { | ||
| 100 | if (checkUrlsSameHost(newUrl, url) !== true) { | ||
| 101 | throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) | ||
| 102 | } | ||
| 103 | |||
| 104 | logger.info('Updating remote AP object %s.', url, lTags(url)) | ||
| 105 | const data = await updater(url, newUrl) | ||
| 106 | |||
| 107 | return { status: 'updated', data } | ||
| 108 | } | ||
| 109 | |||
| 110 | return null | ||
| 111 | } catch (err) { | ||
| 112 | // Does not exist anymore, remove entry | ||
| 113 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
| 114 | return on404OrTombstone() | ||
| 115 | } | ||
| 116 | |||
| 117 | logger.debug('Remote AP object %s is unavailable.', url, lTags(url)) | ||
| 118 | |||
| 119 | const unavailability = await Redis.Instance.addAPUnavailability(url) | ||
| 120 | if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) { | ||
| 121 | logger.info('Removing unavailable AP resource %s.', url, lTags(url)) | ||
| 122 | return on404OrTombstone() | ||
| 123 | } | ||
| 124 | |||
| 125 | return null | ||
| 126 | } | ||
| 127 | } | ||
| 128 | |||
| 129 | function rateOptionsFactory () { | ||
| 130 | return { | ||
| 131 | bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body), | ||
| 132 | |||
| 133 | updater: async (url: string, newUrl: string) => { | ||
| 134 | const rate = await AccountVideoRateModel.loadByUrl(url, undefined) | ||
| 135 | rate.url = newUrl | ||
| 136 | |||
| 137 | const videoId = rate.videoId | ||
| 138 | const type = rate.type | ||
| 139 | |||
| 140 | await rate.save() | ||
| 141 | |||
| 142 | return { videoId, type } | ||
| 143 | }, | ||
| 144 | |||
| 145 | deleter: async (url) => { | ||
| 146 | const rate = await AccountVideoRateModel.loadByUrl(url, undefined) | ||
| 147 | |||
| 148 | const videoId = rate.videoId | ||
| 149 | const type = rate.type | ||
| 150 | |||
| 151 | await rate.destroy() | ||
| 152 | |||
| 153 | return { videoId, type } | ||
| 154 | } | ||
| 155 | } | ||
| 156 | } | ||
| 157 | |||
| 158 | function shareOptionsFactory () { | ||
| 159 | return { | ||
| 160 | bodyValidator: (body: any) => isAnnounceActivityValid(body), | ||
| 161 | |||
| 162 | updater: async (url: string, newUrl: string) => { | ||
| 163 | const share = await VideoShareModel.loadByUrl(url, undefined) | ||
| 164 | share.url = newUrl | ||
| 165 | |||
| 166 | await share.save() | ||
| 167 | |||
| 168 | return undefined | ||
| 169 | }, | ||
| 170 | |||
| 171 | deleter: async (url) => { | ||
| 172 | const share = await VideoShareModel.loadByUrl(url, undefined) | ||
| 173 | |||
| 174 | await share.destroy() | ||
| 175 | |||
| 176 | return undefined | ||
| 177 | } | ||
| 178 | } | ||
| 179 | } | ||
| 180 | |||
| 181 | function commentOptionsFactory () { | ||
| 182 | return { | ||
| 183 | bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body), | ||
| 184 | |||
| 185 | updater: async (url: string, newUrl: string) => { | ||
| 186 | const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) | ||
| 187 | comment.url = newUrl | ||
| 188 | |||
| 189 | await comment.save() | ||
| 190 | |||
| 191 | return undefined | ||
| 192 | }, | ||
| 193 | |||
| 194 | deleter: async (url) => { | ||
| 195 | const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) | ||
| 196 | |||
| 197 | await comment.destroy() | ||
| 198 | |||
| 199 | return undefined | ||
| 200 | } | ||
| 201 | } | ||
| 202 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts deleted file mode 100644 index a68c32ba0..000000000 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ /dev/null | |||
| @@ -1,82 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url' | ||
| 3 | import { ActivitypubFollowPayload } from '@shared/models' | ||
| 4 | import { sanitizeHost } from '../../../helpers/core-utils' | ||
| 5 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
| 6 | import { logger } from '../../../helpers/logger' | ||
| 7 | import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants' | ||
| 8 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 9 | import { ActorModel } from '../../../models/actor/actor' | ||
| 10 | import { ActorFollowModel } from '../../../models/actor/actor-follow' | ||
| 11 | import { MActor, MActorFull } from '../../../types/models' | ||
| 12 | import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors' | ||
| 13 | import { sendFollow } from '../../activitypub/send' | ||
| 14 | import { Notifier } from '../../notifier' | ||
| 15 | |||
| 16 | async function processActivityPubFollow (job: Job) { | ||
| 17 | const payload = job.data as ActivitypubFollowPayload | ||
| 18 | const host = payload.host | ||
| 19 | |||
| 20 | logger.info('Processing ActivityPub follow in job %s.', job.id) | ||
| 21 | |||
| 22 | let targetActor: MActorFull | ||
| 23 | if (!host || host === WEBSERVER.HOST) { | ||
| 24 | targetActor = await ActorModel.loadLocalByName(payload.name) | ||
| 25 | } else { | ||
| 26 | const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) | ||
| 27 | const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) | ||
| 28 | targetActor = await getOrCreateAPActor(actorUrl, 'all') | ||
| 29 | } | ||
| 30 | |||
| 31 | if (payload.assertIsChannel && !targetActor.VideoChannel) { | ||
| 32 | logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host) | ||
| 33 | return | ||
| 34 | } | ||
| 35 | |||
| 36 | const fromActor = await ActorModel.load(payload.followerActorId) | ||
| 37 | |||
| 38 | return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow) | ||
| 39 | } | ||
| 40 | // --------------------------------------------------------------------------- | ||
| 41 | |||
| 42 | export { | ||
| 43 | processActivityPubFollow | ||
| 44 | } | ||
| 45 | |||
| 46 | // --------------------------------------------------------------------------- | ||
| 47 | |||
| 48 | async function follow (fromActor: MActor, targetActor: MActorFull, isAutoFollow = false) { | ||
| 49 | if (fromActor.id === targetActor.id) { | ||
| 50 | throw new Error('Follower is the same as target actor.') | ||
| 51 | } | ||
| 52 | |||
| 53 | // Same server, direct accept | ||
| 54 | const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' | ||
| 55 | |||
| 56 | const actorFollow = await sequelizeTypescript.transaction(async t => { | ||
| 57 | const [ actorFollow ] = await ActorFollowModel.findOrCreateCustom({ | ||
| 58 | byActor: fromActor, | ||
| 59 | state, | ||
| 60 | targetActor, | ||
| 61 | activityId: getLocalActorFollowActivityPubUrl(fromActor, targetActor), | ||
| 62 | transaction: t | ||
| 63 | }) | ||
| 64 | |||
| 65 | // Send a notification to remote server if our follow is not already accepted | ||
| 66 | if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t) | ||
| 67 | |||
| 68 | return actorFollow | ||
| 69 | }) | ||
| 70 | |||
| 71 | const followerFull = await ActorModel.loadFull(fromActor.id) | ||
| 72 | |||
| 73 | const actorFollowFull = Object.assign(actorFollow, { | ||
| 74 | ActorFollowing: targetActor, | ||
| 75 | ActorFollower: followerFull | ||
| 76 | }) | ||
| 77 | |||
| 78 | if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) | ||
| 79 | if (isAutoFollow === true) Notifier.Instance.notifyOfAutoInstanceFollowing(actorFollowFull) | ||
| 80 | |||
| 81 | return actorFollow | ||
| 82 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts deleted file mode 100644 index 8904d086f..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ /dev/null | |||
| @@ -1,49 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send' | ||
| 3 | import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache' | ||
| 4 | import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process' | ||
| 5 | import { ActivitypubHttpBroadcastPayload } from '@shared/models' | ||
| 6 | import { logger } from '../../../helpers/logger' | ||
| 7 | |||
| 8 | // Prefer using a worker thread for HTTP requests because on high load we may have to sign many requests, which can be CPU intensive | ||
| 9 | |||
| 10 | async function processActivityPubHttpSequentialBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) { | ||
| 11 | logger.info('Processing ActivityPub broadcast in job %s.', job.id) | ||
| 12 | |||
| 13 | const requestOptions = await buildRequestOptions(job.data) | ||
| 14 | |||
| 15 | const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) | ||
| 16 | |||
| 17 | return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) | ||
| 18 | } | ||
| 19 | |||
| 20 | async function processActivityPubParallelHttpBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) { | ||
| 21 | logger.info('Processing ActivityPub parallel broadcast in job %s.', job.id) | ||
| 22 | |||
| 23 | const requestOptions = await buildRequestOptions(job.data) | ||
| 24 | |||
| 25 | const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) | ||
| 26 | |||
| 27 | return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) | ||
| 28 | } | ||
| 29 | |||
| 30 | // --------------------------------------------------------------------------- | ||
| 31 | |||
| 32 | export { | ||
| 33 | processActivityPubHttpSequentialBroadcast, | ||
| 34 | processActivityPubParallelHttpBroadcast | ||
| 35 | } | ||
| 36 | |||
| 37 | // --------------------------------------------------------------------------- | ||
| 38 | |||
| 39 | async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) { | ||
| 40 | const body = await computeBody(payload) | ||
| 41 | const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) | ||
| 42 | |||
| 43 | return { | ||
| 44 | method: 'POST' as 'POST', | ||
| 45 | json: body, | ||
| 46 | httpSignature: httpSignatureOptions, | ||
| 47 | headers: buildGlobalHeaders(body) | ||
| 48 | } | ||
| 49 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts deleted file mode 100644 index b6cb3c4a6..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ /dev/null | |||
| @@ -1,41 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { VideoModel } from '../../../models/video/video' | ||
| 5 | import { VideoCommentModel } from '../../../models/video/video-comment' | ||
| 6 | import { VideoShareModel } from '../../../models/video/video-share' | ||
| 7 | import { MVideoFullLight } from '../../../types/models' | ||
| 8 | import { crawlCollectionPage } from '../../activitypub/crawl' | ||
| 9 | import { createAccountPlaylists } from '../../activitypub/playlists' | ||
| 10 | import { processActivities } from '../../activitypub/process' | ||
| 11 | import { addVideoShares } from '../../activitypub/share' | ||
| 12 | import { addVideoComments } from '../../activitypub/video-comments' | ||
| 13 | |||
| 14 | async function processActivityPubHttpFetcher (job: Job) { | ||
| 15 | logger.info('Processing ActivityPub fetcher in job %s.', job.id) | ||
| 16 | |||
| 17 | const payload = job.data as ActivitypubHttpFetcherPayload | ||
| 18 | |||
| 19 | let video: MVideoFullLight | ||
| 20 | if (payload.videoId) video = await VideoModel.loadFull(payload.videoId) | ||
| 21 | |||
| 22 | const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { | ||
| 23 | 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }), | ||
| 24 | 'video-shares': items => addVideoShares(items, video), | ||
| 25 | 'video-comments': items => addVideoComments(items), | ||
| 26 | 'account-playlists': items => createAccountPlaylists(items) | ||
| 27 | } | ||
| 28 | |||
| 29 | const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = { | ||
| 30 | 'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate), | ||
| 31 | 'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) | ||
| 32 | } | ||
| 33 | |||
| 34 | return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type]) | ||
| 35 | } | ||
| 36 | |||
| 37 | // --------------------------------------------------------------------------- | ||
| 38 | |||
| 39 | export { | ||
| 40 | processActivityPubHttpFetcher | ||
| 41 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts deleted file mode 100644 index 50fca3f94..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ /dev/null | |||
| @@ -1,38 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send' | ||
| 3 | import { ActivitypubHttpUnicastPayload } from '@shared/models' | ||
| 4 | import { logger } from '../../../helpers/logger' | ||
| 5 | import { doRequest } from '../../../helpers/requests' | ||
| 6 | import { ActorFollowHealthCache } from '../../actor-follow-health-cache' | ||
| 7 | |||
| 8 | async function processActivityPubHttpUnicast (job: Job) { | ||
| 9 | logger.info('Processing ActivityPub unicast in job %s.', job.id) | ||
| 10 | |||
| 11 | const payload = job.data as ActivitypubHttpUnicastPayload | ||
| 12 | const uri = payload.uri | ||
| 13 | |||
| 14 | const body = await computeBody(payload) | ||
| 15 | const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) | ||
| 16 | |||
| 17 | const options = { | ||
| 18 | method: 'POST' as 'POST', | ||
| 19 | json: body, | ||
| 20 | httpSignature: httpSignatureOptions, | ||
| 21 | headers: buildGlobalHeaders(body) | ||
| 22 | } | ||
| 23 | |||
| 24 | try { | ||
| 25 | await doRequest(uri, options) | ||
| 26 | ActorFollowHealthCache.Instance.updateActorFollowsHealth([ uri ], []) | ||
| 27 | } catch (err) { | ||
| 28 | ActorFollowHealthCache.Instance.updateActorFollowsHealth([], [ uri ]) | ||
| 29 | |||
| 30 | throw err | ||
| 31 | } | ||
| 32 | } | ||
| 33 | |||
| 34 | // --------------------------------------------------------------------------- | ||
| 35 | |||
| 36 | export { | ||
| 37 | processActivityPubHttpUnicast | ||
| 38 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts deleted file mode 100644 index 706bf17fa..000000000 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ /dev/null | |||
| @@ -1,60 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists' | ||
| 3 | import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
| 4 | import { loadVideoByUrl } from '@server/lib/model-loaders' | ||
| 5 | import { RefreshPayload } from '@shared/models' | ||
| 6 | import { logger } from '../../../helpers/logger' | ||
| 7 | import { ActorModel } from '../../../models/actor/actor' | ||
| 8 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
| 9 | import { refreshActorIfNeeded } from '../../activitypub/actors' | ||
| 10 | |||
| 11 | async function refreshAPObject (job: Job) { | ||
| 12 | const payload = job.data as RefreshPayload | ||
| 13 | |||
| 14 | logger.info('Processing AP refresher in job %s for %s.', job.id, payload.url) | ||
| 15 | |||
| 16 | if (payload.type === 'video') return refreshVideo(payload.url) | ||
| 17 | if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url) | ||
| 18 | if (payload.type === 'actor') return refreshActor(payload.url) | ||
| 19 | } | ||
| 20 | |||
| 21 | // --------------------------------------------------------------------------- | ||
| 22 | |||
| 23 | export { | ||
| 24 | refreshAPObject | ||
| 25 | } | ||
| 26 | |||
| 27 | // --------------------------------------------------------------------------- | ||
| 28 | |||
| 29 | async function refreshVideo (videoUrl: string) { | ||
| 30 | const fetchType = 'all' as 'all' | ||
| 31 | const syncParam = { rates: true, shares: true, comments: true } | ||
| 32 | |||
| 33 | const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) | ||
| 34 | if (videoFromDatabase) { | ||
| 35 | const refreshOptions = { | ||
| 36 | video: videoFromDatabase, | ||
| 37 | fetchedType: fetchType, | ||
| 38 | syncParam | ||
| 39 | } | ||
| 40 | |||
| 41 | await refreshVideoIfNeeded(refreshOptions) | ||
| 42 | } | ||
| 43 | } | ||
| 44 | |||
| 45 | async function refreshActor (actorUrl: string) { | ||
| 46 | const fetchType = 'all' as 'all' | ||
| 47 | const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) | ||
| 48 | |||
| 49 | if (actor) { | ||
| 50 | await refreshActorIfNeeded({ actor, fetchedType: fetchType }) | ||
| 51 | } | ||
| 52 | } | ||
| 53 | |||
| 54 | async function refreshVideoPlaylist (playlistUrl: string) { | ||
| 55 | const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl) | ||
| 56 | |||
| 57 | if (playlist) { | ||
| 58 | await refreshVideoPlaylistIfNeeded(playlist) | ||
| 59 | } | ||
| 60 | } | ||
diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts deleted file mode 100644 index 27a2d431b..000000000 --- a/server/lib/job-queue/handlers/actor-keys.ts +++ /dev/null | |||
| @@ -1,20 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors' | ||
| 3 | import { ActorModel } from '@server/models/actor/actor' | ||
| 4 | import { ActorKeysPayload } from '@shared/models' | ||
| 5 | import { logger } from '../../../helpers/logger' | ||
| 6 | |||
| 7 | async function processActorKeys (job: Job) { | ||
| 8 | const payload = job.data as ActorKeysPayload | ||
| 9 | logger.info('Processing actor keys in job %s.', job.id) | ||
| 10 | |||
| 11 | const actor = await ActorModel.load(payload.actorId) | ||
| 12 | |||
| 13 | await generateAndSaveActorKeys(actor) | ||
| 14 | } | ||
| 15 | |||
| 16 | // --------------------------------------------------------------------------- | ||
| 17 | |||
| 18 | export { | ||
| 19 | processActorKeys | ||
| 20 | } | ||
diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts deleted file mode 100644 index ffdd8c5b5..000000000 --- a/server/lib/job-queue/handlers/after-video-channel-import.ts +++ /dev/null | |||
| @@ -1,37 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
| 4 | import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models' | ||
| 5 | |||
| 6 | export async function processAfterVideoChannelImport (job: Job) { | ||
| 7 | const payload = job.data as AfterVideoChannelImportPayload | ||
| 8 | if (!payload.channelSyncId) return | ||
| 9 | |||
| 10 | logger.info('Processing after video channel import in job %s.', job.id) | ||
| 11 | |||
| 12 | const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId) | ||
| 13 | if (!sync) { | ||
| 14 | logger.error('Unknown sync id %d.', payload.channelSyncId) | ||
| 15 | return | ||
| 16 | } | ||
| 17 | |||
| 18 | const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>() | ||
| 19 | |||
| 20 | let errors = 0 | ||
| 21 | let successes = 0 | ||
| 22 | |||
| 23 | for (const value of Object.values(childrenValues)) { | ||
| 24 | if (value.resultType === 'success') successes++ | ||
| 25 | else if (value.resultType === 'error') errors++ | ||
| 26 | } | ||
| 27 | |||
| 28 | if (errors > 0) { | ||
| 29 | sync.state = VideoChannelSyncState.FAILED | ||
| 30 | logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes }) | ||
| 31 | } else { | ||
| 32 | sync.state = VideoChannelSyncState.SYNCED | ||
| 33 | logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes }) | ||
| 34 | } | ||
| 35 | |||
| 36 | await sync.save() | ||
| 37 | } | ||
diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts deleted file mode 100644 index 567bcc076..000000000 --- a/server/lib/job-queue/handlers/email.ts +++ /dev/null | |||
| @@ -1,17 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { EmailPayload } from '@shared/models' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { Emailer } from '../../emailer' | ||
| 5 | |||
| 6 | async function processEmail (job: Job) { | ||
| 7 | const payload = job.data as EmailPayload | ||
| 8 | logger.info('Processing email in job %s.', job.id) | ||
| 9 | |||
| 10 | return Emailer.Instance.sendMail(payload) | ||
| 11 | } | ||
| 12 | |||
| 13 | // --------------------------------------------------------------------------- | ||
| 14 | |||
| 15 | export { | ||
| 16 | processEmail | ||
| 17 | } | ||
diff --git a/server/lib/job-queue/handlers/federate-video.ts b/server/lib/job-queue/handlers/federate-video.ts deleted file mode 100644 index 6aac36741..000000000 --- a/server/lib/job-queue/handlers/federate-video.ts +++ /dev/null | |||
| @@ -1,28 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 3 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 4 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
| 5 | import { VideoModel } from '@server/models/video/video' | ||
| 6 | import { FederateVideoPayload } from '@shared/models' | ||
| 7 | import { logger } from '../../../helpers/logger' | ||
| 8 | |||
| 9 | function processFederateVideo (job: Job) { | ||
| 10 | const payload = job.data as FederateVideoPayload | ||
| 11 | |||
| 12 | logger.info('Processing video federation in job %s.', job.id) | ||
| 13 | |||
| 14 | return retryTransactionWrapper(() => { | ||
| 15 | return sequelizeTypescript.transaction(async t => { | ||
| 16 | const video = await VideoModel.loadFull(payload.videoUUID, t) | ||
| 17 | if (!video) return | ||
| 18 | |||
| 19 | return federateVideoIfNeeded(video, payload.isNewVideo, t) | ||
| 20 | }) | ||
| 21 | }) | ||
| 22 | } | ||
| 23 | |||
| 24 | // --------------------------------------------------------------------------- | ||
| 25 | |||
| 26 | export { | ||
| 27 | processFederateVideo | ||
| 28 | } | ||
diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts deleted file mode 100644 index eea20274a..000000000 --- a/server/lib/job-queue/handlers/generate-storyboard.ts +++ /dev/null | |||
| @@ -1,163 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 4 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | ||
| 5 | import { generateImageFilename, getImageSize } from '@server/helpers/image-utils' | ||
| 6 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 7 | import { deleteFileAndCatch } from '@server/helpers/utils' | ||
| 8 | import { CONFIG } from '@server/initializers/config' | ||
| 9 | import { STORYBOARD } from '@server/initializers/constants' | ||
| 10 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 11 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
| 12 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 13 | import { StoryboardModel } from '@server/models/video/storyboard' | ||
| 14 | import { VideoModel } from '@server/models/video/video' | ||
| 15 | import { MVideo } from '@server/types/models' | ||
| 16 | import { FFmpegImage, isAudioFile } from '@shared/ffmpeg' | ||
| 17 | import { GenerateStoryboardPayload } from '@shared/models' | ||
| 18 | |||
| 19 | const lTagsBase = loggerTagsFactory('storyboard') | ||
| 20 | |||
| 21 | async function processGenerateStoryboard (job: Job): Promise<void> { | ||
| 22 | const payload = job.data as GenerateStoryboardPayload | ||
| 23 | const lTags = lTagsBase(payload.videoUUID) | ||
| 24 | |||
| 25 | logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags) | ||
| 26 | |||
| 27 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) | ||
| 28 | |||
| 29 | try { | ||
| 30 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
| 31 | if (!video) { | ||
| 32 | logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) | ||
| 33 | return | ||
| 34 | } | ||
| 35 | |||
| 36 | const inputFile = video.getMaxQualityFile() | ||
| 37 | |||
| 38 | await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { | ||
| 39 | const isAudio = await isAudioFile(videoPath) | ||
| 40 | |||
| 41 | if (isAudio) { | ||
| 42 | logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) | ||
| 43 | return | ||
| 44 | } | ||
| 45 | |||
| 46 | const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) | ||
| 47 | |||
| 48 | const filename = generateImageFilename() | ||
| 49 | const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) | ||
| 50 | |||
| 51 | const totalSprites = buildTotalSprites(video) | ||
| 52 | if (totalSprites === 0) { | ||
| 53 | logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags) | ||
| 54 | return | ||
| 55 | } | ||
| 56 | |||
| 57 | const spriteDuration = Math.round(video.duration / totalSprites) | ||
| 58 | |||
| 59 | const spritesCount = findGridSize({ | ||
| 60 | toFind: totalSprites, | ||
| 61 | maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT | ||
| 62 | }) | ||
| 63 | |||
| 64 | logger.debug( | ||
| 65 | 'Generating storyboard from video of %s to %s', video.uuid, destination, | ||
| 66 | { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } | ||
| 67 | ) | ||
| 68 | |||
| 69 | await ffmpeg.generateStoryboardFromVideo({ | ||
| 70 | destination, | ||
| 71 | path: videoPath, | ||
| 72 | sprites: { | ||
| 73 | size: STORYBOARD.SPRITE_SIZE, | ||
| 74 | count: spritesCount, | ||
| 75 | duration: spriteDuration | ||
| 76 | } | ||
| 77 | }) | ||
| 78 | |||
| 79 | const imageSize = await getImageSize(destination) | ||
| 80 | |||
| 81 | await retryTransactionWrapper(() => { | ||
| 82 | return sequelizeTypescript.transaction(async transaction => { | ||
| 83 | const videoStillExists = await VideoModel.load(video.id, transaction) | ||
| 84 | if (!videoStillExists) { | ||
| 85 | logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) | ||
| 86 | deleteFileAndCatch(destination) | ||
| 87 | return | ||
| 88 | } | ||
| 89 | |||
| 90 | const existing = await StoryboardModel.loadByVideo(video.id, transaction) | ||
| 91 | if (existing) await existing.destroy({ transaction }) | ||
| 92 | |||
| 93 | await StoryboardModel.create({ | ||
| 94 | filename, | ||
| 95 | totalHeight: imageSize.height, | ||
| 96 | totalWidth: imageSize.width, | ||
| 97 | spriteHeight: STORYBOARD.SPRITE_SIZE.height, | ||
| 98 | spriteWidth: STORYBOARD.SPRITE_SIZE.width, | ||
| 99 | spriteDuration, | ||
| 100 | videoId: video.id | ||
| 101 | }, { transaction }) | ||
| 102 | |||
| 103 | logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags) | ||
| 104 | |||
| 105 | if (payload.federate) { | ||
| 106 | await federateVideoIfNeeded(video, false, transaction) | ||
| 107 | } | ||
| 108 | }) | ||
| 109 | }) | ||
| 110 | }) | ||
| 111 | } finally { | ||
| 112 | inputFileMutexReleaser() | ||
| 113 | } | ||
| 114 | } | ||
| 115 | |||
| 116 | // --------------------------------------------------------------------------- | ||
| 117 | |||
| 118 | export { | ||
| 119 | processGenerateStoryboard | ||
| 120 | } | ||
| 121 | |||
| 122 | function buildTotalSprites (video: MVideo) { | ||
| 123 | const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width | ||
| 124 | const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) | ||
| 125 | |||
| 126 | // We can generate a single line | ||
| 127 | if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites | ||
| 128 | |||
| 129 | return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) | ||
| 130 | } | ||
| 131 | |||
| 132 | function findGridSize (options: { | ||
| 133 | toFind: number | ||
| 134 | maxEdgeCount: number | ||
| 135 | }) { | ||
| 136 | const { toFind, maxEdgeCount } = options | ||
| 137 | |||
| 138 | for (let i = 1; i <= maxEdgeCount; i++) { | ||
| 139 | for (let j = i; j <= maxEdgeCount; j++) { | ||
| 140 | if (toFind === i * j) return { width: j, height: i } | ||
| 141 | } | ||
| 142 | } | ||
| 143 | |||
| 144 | throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`) | ||
| 145 | } | ||
| 146 | |||
| 147 | function findGridFit (value: number, maxMultiplier: number) { | ||
| 148 | for (let i = value; i--; i > 0) { | ||
| 149 | if (!isPrimeWithin(i, maxMultiplier)) return i | ||
| 150 | } | ||
| 151 | |||
| 152 | throw new Error('Could not find prime number below ' + value) | ||
| 153 | } | ||
| 154 | |||
| 155 | function isPrimeWithin (value: number, maxMultiplier: number) { | ||
| 156 | if (value < 2) return false | ||
| 157 | |||
| 158 | for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) { | ||
| 159 | if (value % i === 0 && value / i <= maxMultiplier) return false | ||
| 160 | } | ||
| 161 | |||
| 162 | return true | ||
| 163 | } | ||
diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts deleted file mode 100644 index edf52de0c..000000000 --- a/server/lib/job-queue/handlers/manage-video-torrent.ts +++ /dev/null | |||
| @@ -1,110 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { extractVideo } from '@server/helpers/video' | ||
| 3 | import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent' | ||
| 4 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 5 | import { VideoModel } from '@server/models/video/video' | ||
| 6 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 7 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
| 8 | import { ManageVideoTorrentPayload } from '@shared/models' | ||
| 9 | import { logger } from '../../../helpers/logger' | ||
| 10 | |||
| 11 | async function processManageVideoTorrent (job: Job) { | ||
| 12 | const payload = job.data as ManageVideoTorrentPayload | ||
| 13 | logger.info('Processing torrent in job %s.', job.id) | ||
| 14 | |||
| 15 | if (payload.action === 'create') return doCreateAction(payload) | ||
| 16 | if (payload.action === 'update-metadata') return doUpdateMetadataAction(payload) | ||
| 17 | } | ||
| 18 | |||
| 19 | // --------------------------------------------------------------------------- | ||
| 20 | |||
| 21 | export { | ||
| 22 | processManageVideoTorrent | ||
| 23 | } | ||
| 24 | |||
| 25 | // --------------------------------------------------------------------------- | ||
| 26 | |||
| 27 | async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'create' }) { | ||
| 28 | const [ video, file ] = await Promise.all([ | ||
| 29 | loadVideoOrLog(payload.videoId), | ||
| 30 | loadFileOrLog(payload.videoFileId) | ||
| 31 | ]) | ||
| 32 | |||
| 33 | if (!video || !file) return | ||
| 34 | |||
| 35 | const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 36 | |||
| 37 | try { | ||
| 38 | await video.reload() | ||
| 39 | await file.reload() | ||
| 40 | |||
| 41 | await createTorrentAndSetInfoHash(video, file) | ||
| 42 | |||
| 43 | // Refresh videoFile because the createTorrentAndSetInfoHash could be long | ||
| 44 | const refreshedFile = await VideoFileModel.loadWithVideo(file.id) | ||
| 45 | // File does not exist anymore, remove the generated torrent | ||
| 46 | if (!refreshedFile) return file.removeTorrent() | ||
| 47 | |||
| 48 | refreshedFile.infoHash = file.infoHash | ||
| 49 | refreshedFile.torrentFilename = file.torrentFilename | ||
| 50 | |||
| 51 | await refreshedFile.save() | ||
| 52 | } finally { | ||
| 53 | fileMutexReleaser() | ||
| 54 | } | ||
| 55 | } | ||
| 56 | |||
| 57 | async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) { | ||
| 58 | const [ video, streamingPlaylist, file ] = await Promise.all([ | ||
| 59 | loadVideoOrLog(payload.videoId), | ||
| 60 | loadStreamingPlaylistOrLog(payload.streamingPlaylistId), | ||
| 61 | loadFileOrLog(payload.videoFileId) | ||
| 62 | ]) | ||
| 63 | |||
| 64 | if ((!video && !streamingPlaylist) || !file) return | ||
| 65 | |||
| 66 | const extractedVideo = extractVideo(video || streamingPlaylist) | ||
| 67 | const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid) | ||
| 68 | |||
| 69 | try { | ||
| 70 | await updateTorrentMetadata(video || streamingPlaylist, file) | ||
| 71 | |||
| 72 | await file.save() | ||
| 73 | } finally { | ||
| 74 | fileMutexReleaser() | ||
| 75 | } | ||
| 76 | } | ||
| 77 | |||
| 78 | async function loadVideoOrLog (videoId: number) { | ||
| 79 | if (!videoId) return undefined | ||
| 80 | |||
| 81 | const video = await VideoModel.load(videoId) | ||
| 82 | if (!video) { | ||
| 83 | logger.debug('Do not process torrent for video %d: does not exist anymore.', videoId) | ||
| 84 | } | ||
| 85 | |||
| 86 | return video | ||
| 87 | } | ||
| 88 | |||
| 89 | async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) { | ||
| 90 | if (!streamingPlaylistId) return undefined | ||
| 91 | |||
| 92 | const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) | ||
| 93 | if (!streamingPlaylist) { | ||
| 94 | logger.debug('Do not process torrent for streaming playlist %d: does not exist anymore.', streamingPlaylistId) | ||
| 95 | } | ||
| 96 | |||
| 97 | return streamingPlaylist | ||
| 98 | } | ||
| 99 | |||
| 100 | async function loadFileOrLog (videoFileId: number) { | ||
| 101 | if (!videoFileId) return undefined | ||
| 102 | |||
| 103 | const file = await VideoFileModel.load(videoFileId) | ||
| 104 | |||
| 105 | if (!file) { | ||
| 106 | logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) | ||
| 107 | } | ||
| 108 | |||
| 109 | return file | ||
| 110 | } | ||
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts deleted file mode 100644 index 9a99b6722..000000000 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ /dev/null | |||
| @@ -1,159 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { remove } from 'fs-extra' | ||
| 3 | import { join } from 'path' | ||
| 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' | ||
| 6 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' | ||
| 7 | import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage' | ||
| 8 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | ||
| 9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' | ||
| 11 | import { VideoModel } from '@server/models/video/video' | ||
| 12 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 13 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' | ||
| 14 | import { MoveObjectStoragePayload, VideoState, VideoStorage } from '@shared/models' | ||
| 15 | |||
| 16 | const lTagsBase = loggerTagsFactory('move-object-storage') | ||
| 17 | |||
| 18 | export async function processMoveToObjectStorage (job: Job) { | ||
| 19 | const payload = job.data as MoveObjectStoragePayload | ||
| 20 | logger.info('Moving video %s in job %s.', payload.videoUUID, job.id) | ||
| 21 | |||
| 22 | const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) | ||
| 23 | |||
| 24 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | ||
| 25 | // No video, maybe deleted? | ||
| 26 | if (!video) { | ||
| 27 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID)) | ||
| 28 | fileMutexReleaser() | ||
| 29 | return undefined | ||
| 30 | } | ||
| 31 | |||
| 32 | const lTags = lTagsBase(video.uuid, video.url) | ||
| 33 | |||
| 34 | try { | ||
| 35 | if (video.VideoFiles) { | ||
| 36 | logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags) | ||
| 37 | |||
| 38 | await moveWebVideoFiles(video) | ||
| 39 | } | ||
| 40 | |||
| 41 | if (video.VideoStreamingPlaylists) { | ||
| 42 | logger.debug('Moving HLS playlist of %s.', video.uuid) | ||
| 43 | |||
| 44 | await moveHLSFiles(video) | ||
| 45 | } | ||
| 46 | |||
| 47 | const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') | ||
| 48 | if (pendingMove === 0) { | ||
| 49 | logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags) | ||
| 50 | |||
| 51 | await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) | ||
| 52 | } | ||
| 53 | } catch (err) { | ||
| 54 | await onMoveToObjectStorageFailure(job, err) | ||
| 55 | |||
| 56 | throw err | ||
| 57 | } finally { | ||
| 58 | fileMutexReleaser() | ||
| 59 | } | ||
| 60 | |||
| 61 | return payload.videoUUID | ||
| 62 | } | ||
| 63 | |||
| 64 | export async function onMoveToObjectStorageFailure (job: Job, err: any) { | ||
| 65 | const payload = job.data as MoveObjectStoragePayload | ||
| 66 | |||
| 67 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | ||
| 68 | if (!video) return | ||
| 69 | |||
| 70 | logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) }) | ||
| 71 | |||
| 72 | await moveToFailedMoveToObjectStorageState(video) | ||
| 73 | await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') | ||
| 74 | } | ||
| 75 | |||
| 76 | // --------------------------------------------------------------------------- | ||
| 77 | |||
| 78 | async function moveWebVideoFiles (video: MVideoWithAllFiles) { | ||
| 79 | for (const file of video.VideoFiles) { | ||
| 80 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
| 81 | |||
| 82 | const fileUrl = await storeWebVideoFile(video, file) | ||
| 83 | |||
| 84 | const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) | ||
| 85 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) | ||
| 86 | } | ||
| 87 | } | ||
| 88 | |||
| 89 | async function moveHLSFiles (video: MVideoWithAllFiles) { | ||
| 90 | for (const playlist of video.VideoStreamingPlaylists) { | ||
| 91 | const playlistWithVideo = playlist.withVideo(video) | ||
| 92 | |||
| 93 | for (const file of playlist.VideoFiles) { | ||
| 94 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
| 95 | |||
| 96 | // Resolution playlist | ||
| 97 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | ||
| 98 | await storeHLSFileFromFilename(playlistWithVideo, playlistFilename) | ||
| 99 | |||
| 100 | // Resolution fragmented file | ||
| 101 | const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename) | ||
| 102 | |||
| 103 | const oldPath = join(getHLSDirectory(video), file.filename) | ||
| 104 | |||
| 105 | await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) | ||
| 106 | } | ||
| 107 | } | ||
| 108 | } | ||
| 109 | |||
| 110 | async function doAfterLastJob (options: { | ||
| 111 | video: MVideoWithAllFiles | ||
| 112 | previousVideoState: VideoState | ||
| 113 | isNewVideo: boolean | ||
| 114 | }) { | ||
| 115 | const { video, previousVideoState, isNewVideo } = options | ||
| 116 | |||
| 117 | for (const playlist of video.VideoStreamingPlaylists) { | ||
| 118 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue | ||
| 119 | |||
| 120 | const playlistWithVideo = playlist.withVideo(video) | ||
| 121 | |||
| 122 | // Master playlist | ||
| 123 | playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) | ||
| 124 | // Sha256 segments file | ||
| 125 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) | ||
| 126 | |||
| 127 | playlist.storage = VideoStorage.OBJECT_STORAGE | ||
| 128 | |||
| 129 | playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles) | ||
| 130 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION | ||
| 131 | |||
| 132 | await playlist.save() | ||
| 133 | } | ||
| 134 | |||
| 135 | // Remove empty hls video directory | ||
| 136 | if (video.VideoStreamingPlaylists) { | ||
| 137 | await remove(getHLSDirectory(video)) | ||
| 138 | } | ||
| 139 | |||
| 140 | await moveToNextState({ video, previousVideoState, isNewVideo }) | ||
| 141 | } | ||
| 142 | |||
| 143 | async function onFileMoved (options: { | ||
| 144 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo | ||
| 145 | file: MVideoFile | ||
| 146 | fileUrl: string | ||
| 147 | oldPath: string | ||
| 148 | }) { | ||
| 149 | const { videoOrPlaylist, file, fileUrl, oldPath } = options | ||
| 150 | |||
| 151 | file.fileUrl = fileUrl | ||
| 152 | file.storage = VideoStorage.OBJECT_STORAGE | ||
| 153 | |||
| 154 | await updateTorrentMetadata(videoOrPlaylist, file) | ||
| 155 | await file.save() | ||
| 156 | |||
| 157 | logger.debug('Removing %s because it\'s now on object storage', oldPath) | ||
| 158 | await remove(oldPath) | ||
| 159 | } | ||
diff --git a/server/lib/job-queue/handlers/notify.ts b/server/lib/job-queue/handlers/notify.ts deleted file mode 100644 index 83605396c..000000000 --- a/server/lib/job-queue/handlers/notify.ts +++ /dev/null | |||
| @@ -1,27 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { Notifier } from '@server/lib/notifier' | ||
| 3 | import { VideoModel } from '@server/models/video/video' | ||
| 4 | import { NotifyPayload } from '@shared/models' | ||
| 5 | import { logger } from '../../../helpers/logger' | ||
| 6 | |||
| 7 | async function processNotify (job: Job) { | ||
| 8 | const payload = job.data as NotifyPayload | ||
| 9 | logger.info('Processing %s notification in job %s.', payload.action, job.id) | ||
| 10 | |||
| 11 | if (payload.action === 'new-video') return doNotifyNewVideo(payload) | ||
| 12 | } | ||
| 13 | |||
| 14 | // --------------------------------------------------------------------------- | ||
| 15 | |||
| 16 | export { | ||
| 17 | processNotify | ||
| 18 | } | ||
| 19 | |||
| 20 | // --------------------------------------------------------------------------- | ||
| 21 | |||
| 22 | async function doNotifyNewVideo (payload: NotifyPayload & { action: 'new-video' }) { | ||
| 23 | const refreshedVideo = await VideoModel.loadFull(payload.videoUUID) | ||
| 24 | if (!refreshedVideo) return | ||
| 25 | |||
| 26 | Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo) | ||
| 27 | } | ||
diff --git a/server/lib/job-queue/handlers/transcoding-job-builder.ts b/server/lib/job-queue/handlers/transcoding-job-builder.ts deleted file mode 100644 index 8621b109f..000000000 --- a/server/lib/job-queue/handlers/transcoding-job-builder.ts +++ /dev/null | |||
| @@ -1,48 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { VideoModel } from '@server/models/video/video' | ||
| 5 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 6 | import { pick } from '@shared/core-utils' | ||
| 7 | import { TranscodingJobBuilderPayload } from '@shared/models' | ||
| 8 | import { logger } from '../../../helpers/logger' | ||
| 9 | import { JobQueue } from '../job-queue' | ||
| 10 | |||
| 11 | async function processTranscodingJobBuilder (job: Job) { | ||
| 12 | const payload = job.data as TranscodingJobBuilderPayload | ||
| 13 | |||
| 14 | logger.info('Processing transcoding job builder in job %s.', job.id) | ||
| 15 | |||
| 16 | if (payload.optimizeJob) { | ||
| 17 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
| 18 | const user = await UserModel.loadByVideoId(video.id) | ||
| 19 | const videoFile = video.getMaxQualityFile() | ||
| 20 | |||
| 21 | await createOptimizeOrMergeAudioJobs({ | ||
| 22 | ...pick(payload.optimizeJob, [ 'isNewVideo' ]), | ||
| 23 | |||
| 24 | video, | ||
| 25 | videoFile, | ||
| 26 | user, | ||
| 27 | videoFileAlreadyLocked: false | ||
| 28 | }) | ||
| 29 | } | ||
| 30 | |||
| 31 | for (const job of (payload.jobs || [])) { | ||
| 32 | await JobQueue.Instance.createJob(job) | ||
| 33 | |||
| 34 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') | ||
| 35 | } | ||
| 36 | |||
| 37 | for (const sequentialJobs of (payload.sequentialJobs || [])) { | ||
| 38 | await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs) | ||
| 39 | |||
| 40 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.filter(s => !!s).length) | ||
| 41 | } | ||
| 42 | } | ||
| 43 | |||
| 44 | // --------------------------------------------------------------------------- | ||
| 45 | |||
| 46 | export { | ||
| 47 | processTranscodingJobBuilder | ||
| 48 | } | ||
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts deleted file mode 100644 index 035f88e96..000000000 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ /dev/null | |||
| @@ -1,43 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { CONFIG } from '@server/initializers/config' | ||
| 4 | import { synchronizeChannel } from '@server/lib/sync-channel' | ||
| 5 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
| 6 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
| 7 | import { MChannelSync } from '@server/types/models' | ||
| 8 | import { VideoChannelImportPayload } from '@shared/models' | ||
| 9 | |||
| 10 | export async function processVideoChannelImport (job: Job) { | ||
| 11 | const payload = job.data as VideoChannelImportPayload | ||
| 12 | |||
| 13 | logger.info('Processing video channel import in job %s.', job.id) | ||
| 14 | |||
| 15 | // Channel import requires only http upload to be allowed | ||
| 16 | if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { | ||
| 17 | throw new Error('Cannot import channel as the HTTP upload is disabled') | ||
| 18 | } | ||
| 19 | |||
| 20 | if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { | ||
| 21 | throw new Error('Cannot import channel as the synchronization is disabled') | ||
| 22 | } | ||
| 23 | |||
| 24 | let channelSync: MChannelSync | ||
| 25 | if (payload.partOfChannelSyncId) { | ||
| 26 | channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId) | ||
| 27 | |||
| 28 | if (!channelSync) { | ||
| 29 | throw new Error('Unlnown channel sync specified in videos channel import') | ||
| 30 | } | ||
| 31 | } | ||
| 32 | |||
| 33 | const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) | ||
| 34 | |||
| 35 | logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) | ||
| 36 | |||
| 37 | await synchronizeChannel({ | ||
| 38 | channel: videoChannel, | ||
| 39 | externalChannelUrl: payload.externalChannelUrl, | ||
| 40 | channelSync, | ||
| 41 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT | ||
| 42 | }) | ||
| 43 | } | ||
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts deleted file mode 100644 index d221e8968..000000000 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ /dev/null | |||
| @@ -1,83 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { copy, stat } from 'fs-extra' | ||
| 3 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
| 4 | import { CONFIG } from '@server/initializers/config' | ||
| 5 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
| 6 | import { generateWebVideoFilename } from '@server/lib/paths' | ||
| 7 | import { buildMoveToObjectStorageJob } from '@server/lib/video' | ||
| 8 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 9 | import { VideoModel } from '@server/models/video/video' | ||
| 10 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 11 | import { MVideoFullLight } from '@server/types/models' | ||
| 12 | import { getLowercaseExtension } from '@shared/core-utils' | ||
| 13 | import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' | ||
| 14 | import { VideoFileImportPayload, VideoStorage } from '@shared/models' | ||
| 15 | import { logger } from '../../../helpers/logger' | ||
| 16 | import { JobQueue } from '../job-queue' | ||
| 17 | |||
| 18 | async function processVideoFileImport (job: Job) { | ||
| 19 | const payload = job.data as VideoFileImportPayload | ||
| 20 | logger.info('Processing video file import in job %s.', job.id) | ||
| 21 | |||
| 22 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
| 23 | // No video, maybe deleted? | ||
| 24 | if (!video) { | ||
| 25 | logger.info('Do not process job %d, video does not exist.', job.id) | ||
| 26 | return undefined | ||
| 27 | } | ||
| 28 | |||
| 29 | await updateVideoFile(video, payload.filePath) | ||
| 30 | |||
| 31 | if (CONFIG.OBJECT_STORAGE.ENABLED) { | ||
| 32 | await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState: video.state })) | ||
| 33 | } else { | ||
| 34 | await federateVideoIfNeeded(video, false) | ||
| 35 | } | ||
| 36 | |||
| 37 | return video | ||
| 38 | } | ||
| 39 | |||
| 40 | // --------------------------------------------------------------------------- | ||
| 41 | |||
| 42 | export { | ||
| 43 | processVideoFileImport | ||
| 44 | } | ||
| 45 | |||
| 46 | // --------------------------------------------------------------------------- | ||
| 47 | |||
| 48 | async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | ||
| 49 | const { resolution } = await getVideoStreamDimensionsInfo(inputFilePath) | ||
| 50 | const { size } = await stat(inputFilePath) | ||
| 51 | const fps = await getVideoStreamFPS(inputFilePath) | ||
| 52 | |||
| 53 | const fileExt = getLowercaseExtension(inputFilePath) | ||
| 54 | |||
| 55 | const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === resolution) | ||
| 56 | |||
| 57 | if (currentVideoFile) { | ||
| 58 | // Remove old file and old torrent | ||
| 59 | await video.removeWebVideoFile(currentVideoFile) | ||
| 60 | // Remove the old video file from the array | ||
| 61 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) | ||
| 62 | |||
| 63 | await currentVideoFile.destroy() | ||
| 64 | } | ||
| 65 | |||
| 66 | const newVideoFile = new VideoFileModel({ | ||
| 67 | resolution, | ||
| 68 | extname: fileExt, | ||
| 69 | filename: generateWebVideoFilename(resolution, fileExt), | ||
| 70 | storage: VideoStorage.FILE_SYSTEM, | ||
| 71 | size, | ||
| 72 | fps, | ||
| 73 | videoId: video.id | ||
| 74 | }) | ||
| 75 | |||
| 76 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) | ||
| 77 | await copy(inputFilePath, outputPath) | ||
| 78 | |||
| 79 | video.VideoFiles.push(newVideoFile) | ||
| 80 | await createTorrentAndSetInfoHash(video, newVideoFile) | ||
| 81 | |||
| 82 | await newVideoFile.save() | ||
| 83 | } | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts deleted file mode 100644 index e5cd258d6..000000000 --- a/server/lib/job-queue/handlers/video-import.ts +++ /dev/null | |||
| @@ -1,344 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { move, remove, stat } from 'fs-extra' | ||
| 3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 4 | import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' | ||
| 5 | import { CONFIG } from '@server/initializers/config' | ||
| 6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' | ||
| 7 | import { generateWebVideoFilename } from '@server/lib/paths' | ||
| 8 | import { Hooks } from '@server/lib/plugins/hooks' | ||
| 9 | import { ServerConfigManager } from '@server/lib/server-config-manager' | ||
| 10 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' | ||
| 11 | import { isAbleToUploadVideo } from '@server/lib/user' | ||
| 12 | import { buildMoveToObjectStorageJob } from '@server/lib/video' | ||
| 13 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 14 | import { buildNextVideoState } from '@server/lib/video-state' | ||
| 15 | import { ThumbnailModel } from '@server/models/video/thumbnail' | ||
| 16 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
| 17 | import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | ||
| 18 | import { getLowercaseExtension } from '@shared/core-utils' | ||
| 19 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' | ||
| 20 | import { | ||
| 21 | ThumbnailType, | ||
| 22 | VideoImportPayload, | ||
| 23 | VideoImportPreventExceptionResult, | ||
| 24 | VideoImportState, | ||
| 25 | VideoImportTorrentPayload, | ||
| 26 | VideoImportTorrentPayloadType, | ||
| 27 | VideoImportYoutubeDLPayload, | ||
| 28 | VideoImportYoutubeDLPayloadType, | ||
| 29 | VideoResolution, | ||
| 30 | VideoState | ||
| 31 | } from '@shared/models' | ||
| 32 | import { logger } from '../../../helpers/logger' | ||
| 33 | import { getSecureTorrentName } from '../../../helpers/utils' | ||
| 34 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' | ||
| 35 | import { JOB_TTL } from '../../../initializers/constants' | ||
| 36 | import { sequelizeTypescript } from '../../../initializers/database' | ||
| 37 | import { VideoModel } from '../../../models/video/video' | ||
| 38 | import { VideoFileModel } from '../../../models/video/video-file' | ||
| 39 | import { VideoImportModel } from '../../../models/video/video-import' | ||
| 40 | import { federateVideoIfNeeded } from '../../activitypub/videos' | ||
| 41 | import { Notifier } from '../../notifier' | ||
| 42 | import { generateLocalVideoMiniature } from '../../thumbnail' | ||
| 43 | import { JobQueue } from '../job-queue' | ||
| 44 | |||
| 45 | async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> { | ||
| 46 | const payload = job.data as VideoImportPayload | ||
| 47 | |||
| 48 | const videoImport = await getVideoImportOrDie(payload) | ||
| 49 | if (videoImport.state === VideoImportState.CANCELLED) { | ||
| 50 | logger.info('Do not process import since it has been cancelled', { payload }) | ||
| 51 | return { resultType: 'success' } | ||
| 52 | } | ||
| 53 | |||
| 54 | videoImport.state = VideoImportState.PROCESSING | ||
| 55 | await videoImport.save() | ||
| 56 | |||
| 57 | try { | ||
| 58 | if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload) | ||
| 59 | if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload) | ||
| 60 | |||
| 61 | return { resultType: 'success' } | ||
| 62 | } catch (err) { | ||
| 63 | if (!payload.preventException) throw err | ||
| 64 | |||
| 65 | logger.warn('Catch error in video import to send value to parent job.', { payload, err }) | ||
| 66 | return { resultType: 'error' } | ||
| 67 | } | ||
| 68 | } | ||
| 69 | |||
| 70 | // --------------------------------------------------------------------------- | ||
| 71 | |||
| 72 | export { | ||
| 73 | processVideoImport | ||
| 74 | } | ||
| 75 | |||
| 76 | // --------------------------------------------------------------------------- | ||
| 77 | |||
| 78 | async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) { | ||
| 79 | logger.info('Processing torrent video import in job %s.', job.id) | ||
| 80 | |||
| 81 | const options = { type: payload.type, videoImportId: payload.videoImportId } | ||
| 82 | |||
| 83 | const target = { | ||
| 84 | torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, | ||
| 85 | uri: videoImport.magnetUri | ||
| 86 | } | ||
| 87 | return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options) | ||
| 88 | } | ||
| 89 | |||
| 90 | async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { | ||
| 91 | logger.info('Processing youtubeDL video import in job %s.', job.id) | ||
| 92 | |||
| 93 | const options = { type: payload.type, videoImportId: videoImport.id } | ||
| 94 | |||
| 95 | const youtubeDL = new YoutubeDLWrapper( | ||
| 96 | videoImport.targetUrl, | ||
| 97 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
| 98 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
| 99 | ) | ||
| 100 | |||
| 101 | return processFile( | ||
| 102 | () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']), | ||
| 103 | videoImport, | ||
| 104 | options | ||
| 105 | ) | ||
| 106 | } | ||
| 107 | |||
| 108 | async function getVideoImportOrDie (payload: VideoImportPayload) { | ||
| 109 | const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) | ||
| 110 | if (!videoImport?.Video) { | ||
| 111 | throw new Error(`Cannot import video ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`) | ||
| 112 | } | ||
| 113 | |||
| 114 | return videoImport | ||
| 115 | } | ||
| 116 | |||
| 117 | type ProcessFileOptions = { | ||
| 118 | type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType | ||
| 119 | videoImportId: number | ||
| 120 | } | ||
| 121 | async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { | ||
| 122 | let tempVideoPath: string | ||
| 123 | let videoFile: VideoFileModel | ||
| 124 | |||
| 125 | try { | ||
| 126 | // Download video from youtubeDL | ||
| 127 | tempVideoPath = await downloader() | ||
| 128 | |||
| 129 | // Get information about this video | ||
| 130 | const stats = await stat(tempVideoPath) | ||
| 131 | const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size) | ||
| 132 | if (isAble === false) { | ||
| 133 | throw new Error('The user video quota is exceeded with this video to import.') | ||
| 134 | } | ||
| 135 | |||
| 136 | const probe = await ffprobePromise(tempVideoPath) | ||
| 137 | |||
| 138 | const { resolution } = await isAudioFile(tempVideoPath, probe) | ||
| 139 | ? { resolution: VideoResolution.H_NOVIDEO } | ||
| 140 | : await getVideoStreamDimensionsInfo(tempVideoPath, probe) | ||
| 141 | |||
| 142 | const fps = await getVideoStreamFPS(tempVideoPath, probe) | ||
| 143 | const duration = await getVideoStreamDuration(tempVideoPath, probe) | ||
| 144 | |||
| 145 | // Prepare video file object for creation in database | ||
| 146 | const fileExt = getLowercaseExtension(tempVideoPath) | ||
| 147 | const videoFileData = { | ||
| 148 | extname: fileExt, | ||
| 149 | resolution, | ||
| 150 | size: stats.size, | ||
| 151 | filename: generateWebVideoFilename(resolution, fileExt), | ||
| 152 | fps, | ||
| 153 | videoId: videoImport.videoId | ||
| 154 | } | ||
| 155 | videoFile = new VideoFileModel(videoFileData) | ||
| 156 | |||
| 157 | const hookName = options.type === 'youtube-dl' | ||
| 158 | ? 'filter:api.video.post-import-url.accept.result' | ||
| 159 | : 'filter:api.video.post-import-torrent.accept.result' | ||
| 160 | |||
| 161 | // Check we accept this video | ||
| 162 | const acceptParameters = { | ||
| 163 | videoImport, | ||
| 164 | video: videoImport.Video, | ||
| 165 | videoFilePath: tempVideoPath, | ||
| 166 | videoFile, | ||
| 167 | user: videoImport.User | ||
| 168 | } | ||
| 169 | const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName) | ||
| 170 | |||
| 171 | if (acceptedResult.accepted !== true) { | ||
| 172 | logger.info('Refused imported video.', { acceptedResult, acceptParameters }) | ||
| 173 | |||
| 174 | videoImport.state = VideoImportState.REJECTED | ||
| 175 | await videoImport.save() | ||
| 176 | |||
| 177 | throw new Error(acceptedResult.errorMessage) | ||
| 178 | } | ||
| 179 | |||
| 180 | // Video is accepted, resuming preparation | ||
| 181 | const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid) | ||
| 182 | |||
| 183 | try { | ||
| 184 | const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile) | ||
| 185 | |||
| 186 | // Move file | ||
| 187 | const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) | ||
| 188 | await move(tempVideoPath, videoDestFile) | ||
| 189 | |||
| 190 | tempVideoPath = null // This path is not used anymore | ||
| 191 | |||
| 192 | let { | ||
| 193 | miniatureModel: thumbnailModel, | ||
| 194 | miniatureJSONSave: thumbnailSave | ||
| 195 | } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE) | ||
| 196 | |||
| 197 | let { | ||
| 198 | miniatureModel: previewModel, | ||
| 199 | miniatureJSONSave: previewSave | ||
| 200 | } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW) | ||
| 201 | |||
| 202 | // Create torrent | ||
| 203 | await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) | ||
| 204 | |||
| 205 | const videoFileSave = videoFile.toJSON() | ||
| 206 | |||
| 207 | const { videoImportUpdated, video } = await retryTransactionWrapper(() => { | ||
| 208 | return sequelizeTypescript.transaction(async t => { | ||
| 209 | // Refresh video | ||
| 210 | const video = await VideoModel.load(videoImportWithFiles.videoId, t) | ||
| 211 | if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.') | ||
| 212 | |||
| 213 | await videoFile.save({ transaction: t }) | ||
| 214 | |||
| 215 | // Update video DB object | ||
| 216 | video.duration = duration | ||
| 217 | video.state = buildNextVideoState(video.state) | ||
| 218 | await video.save({ transaction: t }) | ||
| 219 | |||
| 220 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) | ||
| 221 | if (previewModel) await video.addAndSaveThumbnail(previewModel, t) | ||
| 222 | |||
| 223 | // Now we can federate the video (reload from database, we need more attributes) | ||
| 224 | const videoForFederation = await VideoModel.loadFull(video.uuid, t) | ||
| 225 | await federateVideoIfNeeded(videoForFederation, true, t) | ||
| 226 | |||
| 227 | // Update video import object | ||
| 228 | videoImportWithFiles.state = VideoImportState.SUCCESS | ||
| 229 | const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport | ||
| 230 | |||
| 231 | logger.info('Video %s imported.', video.uuid) | ||
| 232 | |||
| 233 | return { videoImportUpdated, video: videoForFederation } | ||
| 234 | }).catch(err => { | ||
| 235 | // Reset fields | ||
| 236 | if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave) | ||
| 237 | if (previewModel) previewModel = new ThumbnailModel(previewSave) | ||
| 238 | |||
| 239 | videoFile = new VideoFileModel(videoFileSave) | ||
| 240 | |||
| 241 | throw err | ||
| 242 | }) | ||
| 243 | }) | ||
| 244 | |||
| 245 | await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User, videoFileAlreadyLocked: true }) | ||
| 246 | } finally { | ||
| 247 | videoFileLockReleaser() | ||
| 248 | } | ||
| 249 | } catch (err) { | ||
| 250 | await onImportError(err, tempVideoPath, videoImport) | ||
| 251 | |||
| 252 | throw err | ||
| 253 | } | ||
| 254 | } | ||
| 255 | |||
| 256 | async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> { | ||
| 257 | // Refresh video, privacy may have changed | ||
| 258 | const video = await videoImport.Video.reload() | ||
| 259 | const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) | ||
| 260 | |||
| 261 | return Object.assign(videoImport, { Video: videoWithFiles }) | ||
| 262 | } | ||
| 263 | |||
| 264 | async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles, videoFile: MVideoFile, thumbnailType: ThumbnailType) { | ||
| 265 | // Generate miniature if the import did not created it | ||
| 266 | const needsMiniature = thumbnailType === ThumbnailType.MINIATURE | ||
| 267 | ? !videoImportWithFiles.Video.getMiniature() | ||
| 268 | : !videoImportWithFiles.Video.getPreview() | ||
| 269 | |||
| 270 | if (!needsMiniature) { | ||
| 271 | return { | ||
| 272 | miniatureModel: null, | ||
| 273 | miniatureJSONSave: null | ||
| 274 | } | ||
| 275 | } | ||
| 276 | |||
| 277 | const miniatureModel = await generateLocalVideoMiniature({ | ||
| 278 | video: videoImportWithFiles.Video, | ||
| 279 | videoFile, | ||
| 280 | type: thumbnailType | ||
| 281 | }) | ||
| 282 | const miniatureJSONSave = miniatureModel.toJSON() | ||
| 283 | |||
| 284 | return { | ||
| 285 | miniatureModel, | ||
| 286 | miniatureJSONSave | ||
| 287 | } | ||
| 288 | } | ||
| 289 | |||
| 290 | async function afterImportSuccess (options: { | ||
| 291 | videoImport: MVideoImport | ||
| 292 | video: MVideoFullLight | ||
| 293 | videoFile: MVideoFile | ||
| 294 | user: MUserId | ||
| 295 | videoFileAlreadyLocked: boolean | ||
| 296 | }) { | ||
| 297 | const { video, videoFile, videoImport, user, videoFileAlreadyLocked } = options | ||
| 298 | |||
| 299 | Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true }) | ||
| 300 | |||
| 301 | if (video.isBlacklisted()) { | ||
| 302 | const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video }) | ||
| 303 | |||
| 304 | Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) | ||
| 305 | } else { | ||
| 306 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
| 307 | } | ||
| 308 | |||
| 309 | // Generate the storyboard in the job queue, and don't forget to federate an update after | ||
| 310 | await JobQueue.Instance.createJob({ | ||
| 311 | type: 'generate-video-storyboard' as 'generate-video-storyboard', | ||
| 312 | payload: { | ||
| 313 | videoUUID: video.uuid, | ||
| 314 | federate: true | ||
| 315 | } | ||
| 316 | }) | ||
| 317 | |||
| 318 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
| 319 | await JobQueue.Instance.createJob( | ||
| 320 | await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) | ||
| 321 | ) | ||
| 322 | return | ||
| 323 | } | ||
| 324 | |||
| 325 | if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? | ||
| 326 | await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user, videoFileAlreadyLocked }) | ||
| 327 | } | ||
| 328 | } | ||
| 329 | |||
| 330 | async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) { | ||
| 331 | try { | ||
| 332 | if (tempVideoPath) await remove(tempVideoPath) | ||
| 333 | } catch (errUnlink) { | ||
| 334 | logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) | ||
| 335 | } | ||
| 336 | |||
| 337 | videoImport.error = err.message | ||
| 338 | if (videoImport.state !== VideoImportState.REJECTED) { | ||
| 339 | videoImport.state = VideoImportState.FAILED | ||
| 340 | } | ||
| 341 | await videoImport.save() | ||
| 342 | |||
| 343 | Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false }) | ||
| 344 | } | ||
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts deleted file mode 100644 index 070d1d7a2..000000000 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ /dev/null | |||
| @@ -1,279 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { readdir, remove } from 'fs-extra' | ||
| 3 | import { join } from 'path' | ||
| 4 | import { peertubeTruncate } from '@server/helpers/core-utils' | ||
| 5 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
| 6 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | ||
| 7 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
| 8 | import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' | ||
| 9 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' | ||
| 10 | import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail' | ||
| 11 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' | ||
| 12 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 13 | import { moveToNextState } from '@server/lib/video-state' | ||
| 14 | import { VideoModel } from '@server/models/video/video' | ||
| 15 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' | ||
| 16 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 17 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
| 18 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
| 19 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' | ||
| 20 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
| 21 | import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' | ||
| 22 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' | ||
| 23 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' | ||
| 24 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | ||
| 25 | import { JobQueue } from '../job-queue' | ||
| 26 | |||
| 27 | const lTags = loggerTagsFactory('live', 'job') | ||
| 28 | |||
| 29 | async function processVideoLiveEnding (job: Job) { | ||
| 30 | const payload = job.data as VideoLiveEndingPayload | ||
| 31 | |||
| 32 | logger.info('Processing video live ending for %s.', payload.videoId, { payload, ...lTags() }) | ||
| 33 | |||
| 34 | function logError () { | ||
| 35 | logger.warn('Video live %d does not exist anymore. Cannot process live ending.', payload.videoId, lTags()) | ||
| 36 | } | ||
| 37 | |||
| 38 | const video = await VideoModel.load(payload.videoId) | ||
| 39 | const live = await VideoLiveModel.loadByVideoId(payload.videoId) | ||
| 40 | const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) | ||
| 41 | |||
| 42 | if (!video || !live || !liveSession) { | ||
| 43 | logError() | ||
| 44 | return | ||
| 45 | } | ||
| 46 | |||
| 47 | const permanentLive = live.permanentLive | ||
| 48 | |||
| 49 | liveSession.endingProcessed = true | ||
| 50 | await liveSession.save() | ||
| 51 | |||
| 52 | if (liveSession.saveReplay !== true) { | ||
| 53 | return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) | ||
| 54 | } | ||
| 55 | |||
| 56 | if (permanentLive) { | ||
| 57 | await saveReplayToExternalVideo({ | ||
| 58 | liveVideo: video, | ||
| 59 | liveSession, | ||
| 60 | publishedAt: payload.publishedAt, | ||
| 61 | replayDirectory: payload.replayDirectory | ||
| 62 | }) | ||
| 63 | |||
| 64 | return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) | ||
| 65 | } | ||
| 66 | |||
| 67 | return replaceLiveByReplay({ | ||
| 68 | video, | ||
| 69 | liveSession, | ||
| 70 | live, | ||
| 71 | permanentLive, | ||
| 72 | replayDirectory: payload.replayDirectory | ||
| 73 | }) | ||
| 74 | } | ||
| 75 | |||
| 76 | // --------------------------------------------------------------------------- | ||
| 77 | |||
| 78 | export { | ||
| 79 | processVideoLiveEnding | ||
| 80 | } | ||
| 81 | |||
| 82 | // --------------------------------------------------------------------------- | ||
| 83 | |||
| 84 | async function saveReplayToExternalVideo (options: { | ||
| 85 | liveVideo: MVideo | ||
| 86 | liveSession: MVideoLiveSession | ||
| 87 | publishedAt: string | ||
| 88 | replayDirectory: string | ||
| 89 | }) { | ||
| 90 | const { liveVideo, liveSession, publishedAt, replayDirectory } = options | ||
| 91 | |||
| 92 | const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) | ||
| 93 | |||
| 94 | const videoNameSuffix = ` - ${new Date(publishedAt).toLocaleString()}` | ||
| 95 | const truncatedVideoName = peertubeTruncate(liveVideo.name, { | ||
| 96 | length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max - videoNameSuffix.length | ||
| 97 | }) | ||
| 98 | |||
| 99 | const replayVideo = new VideoModel({ | ||
| 100 | name: truncatedVideoName + videoNameSuffix, | ||
| 101 | isLive: false, | ||
| 102 | state: VideoState.TO_TRANSCODE, | ||
| 103 | duration: 0, | ||
| 104 | |||
| 105 | remote: liveVideo.remote, | ||
| 106 | category: liveVideo.category, | ||
| 107 | licence: liveVideo.licence, | ||
| 108 | language: liveVideo.language, | ||
| 109 | commentsEnabled: liveVideo.commentsEnabled, | ||
| 110 | downloadEnabled: liveVideo.downloadEnabled, | ||
| 111 | waitTranscoding: true, | ||
| 112 | nsfw: liveVideo.nsfw, | ||
| 113 | description: liveVideo.description, | ||
| 114 | support: liveVideo.support, | ||
| 115 | privacy: replaySettings.privacy, | ||
| 116 | channelId: liveVideo.channelId | ||
| 117 | }) as MVideoWithAllFiles | ||
| 118 | |||
| 119 | replayVideo.Thumbnails = [] | ||
| 120 | replayVideo.VideoFiles = [] | ||
| 121 | replayVideo.VideoStreamingPlaylists = [] | ||
| 122 | |||
| 123 | replayVideo.url = getLocalVideoActivityPubUrl(replayVideo) | ||
| 124 | |||
| 125 | await replayVideo.save() | ||
| 126 | |||
| 127 | liveSession.replayVideoId = replayVideo.id | ||
| 128 | await liveSession.save() | ||
| 129 | |||
| 130 | // If live is blacklisted, also blacklist the replay | ||
| 131 | const blacklist = await VideoBlacklistModel.loadByVideoId(liveVideo.id) | ||
| 132 | if (blacklist) { | ||
| 133 | await VideoBlacklistModel.create({ | ||
| 134 | videoId: replayVideo.id, | ||
| 135 | unfederated: blacklist.unfederated, | ||
| 136 | reason: blacklist.reason, | ||
| 137 | type: blacklist.type | ||
| 138 | }) | ||
| 139 | } | ||
| 140 | |||
| 141 | await assignReplayFilesToVideo({ video: replayVideo, replayDirectory }) | ||
| 142 | |||
| 143 | await remove(replayDirectory) | ||
| 144 | |||
| 145 | for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { | ||
| 146 | const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) | ||
| 147 | await replayVideo.addAndSaveThumbnail(image) | ||
| 148 | } | ||
| 149 | |||
| 150 | await moveToNextState({ video: replayVideo, isNewVideo: true }) | ||
| 151 | |||
| 152 | await createStoryboardJob(replayVideo) | ||
| 153 | } | ||
| 154 | |||
| 155 | async function replaceLiveByReplay (options: { | ||
| 156 | video: MVideo | ||
| 157 | liveSession: MVideoLiveSession | ||
| 158 | live: MVideoLive | ||
| 159 | permanentLive: boolean | ||
| 160 | replayDirectory: string | ||
| 161 | }) { | ||
| 162 | const { video, liveSession, live, permanentLive, replayDirectory } = options | ||
| 163 | |||
| 164 | const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) | ||
| 165 | const videoWithFiles = await VideoModel.loadFull(video.id) | ||
| 166 | const hlsPlaylist = videoWithFiles.getHLSPlaylist() | ||
| 167 | |||
| 168 | await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist) | ||
| 169 | |||
| 170 | await live.destroy() | ||
| 171 | |||
| 172 | videoWithFiles.isLive = false | ||
| 173 | videoWithFiles.privacy = replaySettings.privacy | ||
| 174 | videoWithFiles.waitTranscoding = true | ||
| 175 | videoWithFiles.state = VideoState.TO_TRANSCODE | ||
| 176 | |||
| 177 | await videoWithFiles.save() | ||
| 178 | |||
| 179 | liveSession.replayVideoId = videoWithFiles.id | ||
| 180 | await liveSession.save() | ||
| 181 | |||
| 182 | await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) | ||
| 183 | |||
| 184 | // Reset playlist | ||
| 185 | hlsPlaylist.VideoFiles = [] | ||
| 186 | hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename() | ||
| 187 | hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename() | ||
| 188 | await hlsPlaylist.save() | ||
| 189 | |||
| 190 | await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) | ||
| 191 | |||
| 192 | // Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay | ||
| 193 | if (permanentLive) { // Remove session replay | ||
| 194 | await remove(replayDirectory) | ||
| 195 | } else { // We won't stream again in this live, we can delete the base replay directory | ||
| 196 | await remove(getLiveReplayBaseDirectory(videoWithFiles)) | ||
| 197 | } | ||
| 198 | |||
| 199 | // Regenerate the thumbnail & preview? | ||
| 200 | await regenerateMiniaturesIfNeeded(videoWithFiles) | ||
| 201 | |||
| 202 | // We consider this is a new video | ||
| 203 | await moveToNextState({ video: videoWithFiles, isNewVideo: true }) | ||
| 204 | |||
| 205 | await createStoryboardJob(videoWithFiles) | ||
| 206 | } | ||
| 207 | |||
| 208 | async function assignReplayFilesToVideo (options: { | ||
| 209 | video: MVideo | ||
| 210 | replayDirectory: string | ||
| 211 | }) { | ||
| 212 | const { video, replayDirectory } = options | ||
| 213 | |||
| 214 | const concatenatedTsFiles = await readdir(replayDirectory) | ||
| 215 | |||
| 216 | for (const concatenatedTsFile of concatenatedTsFiles) { | ||
| 217 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 218 | await video.reload() | ||
| 219 | |||
| 220 | const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) | ||
| 221 | |||
| 222 | const probe = await ffprobePromise(concatenatedTsFilePath) | ||
| 223 | const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) | ||
| 224 | const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) | ||
| 225 | const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe) | ||
| 226 | |||
| 227 | try { | ||
| 228 | await generateHlsPlaylistResolutionFromTS({ | ||
| 229 | video, | ||
| 230 | inputFileMutexReleaser, | ||
| 231 | concatenatedTsFilePath, | ||
| 232 | resolution, | ||
| 233 | fps, | ||
| 234 | isAAC: audioStream?.codec_name === 'aac' | ||
| 235 | }) | ||
| 236 | } catch (err) { | ||
| 237 | logger.error('Cannot generate HLS playlist resolution from TS files.', { err }) | ||
| 238 | } | ||
| 239 | |||
| 240 | inputFileMutexReleaser() | ||
| 241 | } | ||
| 242 | |||
| 243 | return video | ||
| 244 | } | ||
| 245 | |||
| 246 | async function cleanupLiveAndFederate (options: { | ||
| 247 | video: MVideo | ||
| 248 | permanentLive: boolean | ||
| 249 | streamingPlaylistId: number | ||
| 250 | }) { | ||
| 251 | const { permanentLive, video, streamingPlaylistId } = options | ||
| 252 | |||
| 253 | const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) | ||
| 254 | |||
| 255 | if (streamingPlaylist) { | ||
| 256 | if (permanentLive) { | ||
| 257 | await cleanupAndDestroyPermanentLive(video, streamingPlaylist) | ||
| 258 | } else { | ||
| 259 | await cleanupUnsavedNormalLive(video, streamingPlaylist) | ||
| 260 | } | ||
| 261 | } | ||
| 262 | |||
| 263 | try { | ||
| 264 | const fullVideo = await VideoModel.loadFull(video.id) | ||
| 265 | return federateVideoIfNeeded(fullVideo, false, undefined) | ||
| 266 | } catch (err) { | ||
| 267 | logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) | ||
| 268 | } | ||
| 269 | } | ||
| 270 | |||
| 271 | function createStoryboardJob (video: MVideo) { | ||
| 272 | return JobQueue.Instance.createJob({ | ||
| 273 | type: 'generate-video-storyboard' as 'generate-video-storyboard', | ||
| 274 | payload: { | ||
| 275 | videoUUID: video.uuid, | ||
| 276 | federate: true | ||
| 277 | } | ||
| 278 | }) | ||
| 279 | } | ||
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts deleted file mode 100644 index bac99fdb7..000000000 --- a/server/lib/job-queue/handlers/video-redundancy.ts +++ /dev/null | |||
| @@ -1,17 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' | ||
| 3 | import { VideoRedundancyPayload } from '@shared/models' | ||
| 4 | import { logger } from '../../../helpers/logger' | ||
| 5 | |||
| 6 | async function processVideoRedundancy (job: Job) { | ||
| 7 | const payload = job.data as VideoRedundancyPayload | ||
| 8 | logger.info('Processing video redundancy in job %s.', job.id) | ||
| 9 | |||
| 10 | return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) | ||
| 11 | } | ||
| 12 | |||
| 13 | // --------------------------------------------------------------------------- | ||
| 14 | |||
| 15 | export { | ||
| 16 | processVideoRedundancy | ||
| 17 | } | ||
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts deleted file mode 100644 index caf051bfa..000000000 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ /dev/null | |||
| @@ -1,180 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { remove } from 'fs-extra' | ||
| 3 | import { join } from 'path' | ||
| 4 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | ||
| 5 | import { CONFIG } from '@server/initializers/config' | ||
| 6 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' | ||
| 7 | import { isAbleToUploadVideo } from '@server/lib/user' | ||
| 8 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 9 | import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' | ||
| 10 | import { UserModel } from '@server/models/user/user' | ||
| 11 | import { VideoModel } from '@server/models/video/video' | ||
| 12 | import { MVideo, MVideoFullLight } from '@server/types/models' | ||
| 13 | import { pick } from '@shared/core-utils' | ||
| 14 | import { buildUUID } from '@shared/extra-utils' | ||
| 15 | import { FFmpegEdition } from '@shared/ffmpeg' | ||
| 16 | import { | ||
| 17 | VideoStudioEditionPayload, | ||
| 18 | VideoStudioTask, | ||
| 19 | VideoStudioTaskCutPayload, | ||
| 20 | VideoStudioTaskIntroPayload, | ||
| 21 | VideoStudioTaskOutroPayload, | ||
| 22 | VideoStudioTaskPayload, | ||
| 23 | VideoStudioTaskWatermarkPayload | ||
| 24 | } from '@shared/models' | ||
| 25 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | ||
| 26 | |||
| 27 | const lTagsBase = loggerTagsFactory('video-studio') | ||
| 28 | |||
| 29 | async function processVideoStudioEdition (job: Job) { | ||
| 30 | const payload = job.data as VideoStudioEditionPayload | ||
| 31 | const lTags = lTagsBase(payload.videoUUID) | ||
| 32 | |||
| 33 | logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) | ||
| 34 | |||
| 35 | try { | ||
| 36 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
| 37 | |||
| 38 | // No video, maybe deleted? | ||
| 39 | if (!video) { | ||
| 40 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) | ||
| 41 | |||
| 42 | await safeCleanupStudioTMPFiles(payload.tasks) | ||
| 43 | return undefined | ||
| 44 | } | ||
| 45 | |||
| 46 | await checkUserQuotaOrThrow(video, payload) | ||
| 47 | |||
| 48 | const inputFile = video.getMaxQualityFile() | ||
| 49 | |||
| 50 | const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { | ||
| 51 | let tmpInputFilePath: string | ||
| 52 | let outputPath: string | ||
| 53 | |||
| 54 | for (const task of payload.tasks) { | ||
| 55 | const outputFilename = buildUUID() + inputFile.extname | ||
| 56 | outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) | ||
| 57 | |||
| 58 | await processTask({ | ||
| 59 | inputPath: tmpInputFilePath ?? originalFilePath, | ||
| 60 | video, | ||
| 61 | outputPath, | ||
| 62 | task, | ||
| 63 | lTags | ||
| 64 | }) | ||
| 65 | |||
| 66 | if (tmpInputFilePath) await remove(tmpInputFilePath) | ||
| 67 | |||
| 68 | // For the next iteration | ||
| 69 | tmpInputFilePath = outputPath | ||
| 70 | } | ||
| 71 | |||
| 72 | return outputPath | ||
| 73 | }) | ||
| 74 | |||
| 75 | logger.info('Video edition ended for video %s.', video.uuid, lTags) | ||
| 76 | |||
| 77 | await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks }) | ||
| 78 | } catch (err) { | ||
| 79 | await safeCleanupStudioTMPFiles(payload.tasks) | ||
| 80 | |||
| 81 | throw err | ||
| 82 | } | ||
| 83 | } | ||
| 84 | |||
| 85 | // --------------------------------------------------------------------------- | ||
| 86 | |||
| 87 | export { | ||
| 88 | processVideoStudioEdition | ||
| 89 | } | ||
| 90 | |||
| 91 | // --------------------------------------------------------------------------- | ||
| 92 | |||
| 93 | type TaskProcessorOptions <T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = { | ||
| 94 | inputPath: string | ||
| 95 | outputPath: string | ||
| 96 | video: MVideo | ||
| 97 | task: T | ||
| 98 | lTags: { tags: string[] } | ||
| 99 | } | ||
| 100 | |||
| 101 | const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = { | ||
| 102 | 'add-intro': processAddIntroOutro, | ||
| 103 | 'add-outro': processAddIntroOutro, | ||
| 104 | 'cut': processCut, | ||
| 105 | 'add-watermark': processAddWatermark | ||
| 106 | } | ||
| 107 | |||
| 108 | async function processTask (options: TaskProcessorOptions) { | ||
| 109 | const { video, task, lTags } = options | ||
| 110 | |||
| 111 | logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags }) | ||
| 112 | |||
| 113 | const processor = taskProcessors[options.task.name] | ||
| 114 | if (!process) throw new Error('Unknown task ' + task.name) | ||
| 115 | |||
| 116 | return processor(options) | ||
| 117 | } | ||
| 118 | |||
| 119 | function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) { | ||
| 120 | const { task, lTags } = options | ||
| 121 | |||
| 122 | logger.debug('Will add intro/outro to the video.', { options, ...lTags }) | ||
| 123 | |||
| 124 | return buildFFmpegEdition().addIntroOutro({ | ||
| 125 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
| 126 | |||
| 127 | introOutroPath: task.options.file, | ||
| 128 | type: task.name === 'add-intro' | ||
| 129 | ? 'intro' | ||
| 130 | : 'outro' | ||
| 131 | }) | ||
| 132 | } | ||
| 133 | |||
| 134 | function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) { | ||
| 135 | const { task, lTags } = options | ||
| 136 | |||
| 137 | logger.debug('Will cut the video.', { options, ...lTags }) | ||
| 138 | |||
| 139 | return buildFFmpegEdition().cutVideo({ | ||
| 140 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
| 141 | |||
| 142 | start: task.options.start, | ||
| 143 | end: task.options.end | ||
| 144 | }) | ||
| 145 | } | ||
| 146 | |||
| 147 | function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) { | ||
| 148 | const { task, lTags } = options | ||
| 149 | |||
| 150 | logger.debug('Will add watermark to the video.', { options, ...lTags }) | ||
| 151 | |||
| 152 | return buildFFmpegEdition().addWatermark({ | ||
| 153 | ...pick(options, [ 'inputPath', 'outputPath' ]), | ||
| 154 | |||
| 155 | watermarkPath: task.options.file, | ||
| 156 | |||
| 157 | videoFilters: { | ||
| 158 | watermarkSizeRatio: task.options.watermarkSizeRatio, | ||
| 159 | horitonzalMarginRatio: task.options.horitonzalMarginRatio, | ||
| 160 | verticalMarginRatio: task.options.verticalMarginRatio | ||
| 161 | } | ||
| 162 | }) | ||
| 163 | } | ||
| 164 | |||
| 165 | // --------------------------------------------------------------------------- | ||
| 166 | |||
| 167 | async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) { | ||
| 168 | const user = await UserModel.loadByVideoId(video.id) | ||
| 169 | |||
| 170 | const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file | ||
| 171 | |||
| 172 | const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder) | ||
| 173 | if (await isAbleToUploadVideo(user.id, additionalBytes) === false) { | ||
| 174 | throw new Error('Quota exceeded for this user to edit the video') | ||
| 175 | } | ||
| 176 | } | ||
| 177 | |||
| 178 | function buildFFmpegEdition () { | ||
| 179 | return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) | ||
| 180 | } | ||
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts deleted file mode 100644 index 1c8f4fd9f..000000000 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ /dev/null | |||
| @@ -1,150 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | ||
| 3 | import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding' | ||
| 4 | import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding' | ||
| 5 | import { removeAllWebVideoFiles } from '@server/lib/video-file' | ||
| 6 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 7 | import { moveToFailedTranscodingState } from '@server/lib/video-state' | ||
| 8 | import { UserModel } from '@server/models/user/user' | ||
| 9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 10 | import { MUser, MUserId, MVideoFullLight } from '@server/types/models' | ||
| 11 | import { | ||
| 12 | HLSTranscodingPayload, | ||
| 13 | MergeAudioTranscodingPayload, | ||
| 14 | NewWebVideoResolutionTranscodingPayload, | ||
| 15 | OptimizeTranscodingPayload, | ||
| 16 | VideoTranscodingPayload | ||
| 17 | } from '@shared/models' | ||
| 18 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | ||
| 19 | import { VideoModel } from '../../../models/video/video' | ||
| 20 | |||
| 21 | type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void> | ||
| 22 | |||
| 23 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { | ||
| 24 | 'new-resolution-to-hls': handleHLSJob, | ||
| 25 | 'new-resolution-to-web-video': handleNewWebVideoResolutionJob, | ||
| 26 | 'merge-audio-to-web-video': handleWebVideoMergeAudioJob, | ||
| 27 | 'optimize-to-web-video': handleWebVideoOptimizeJob | ||
| 28 | } | ||
| 29 | |||
| 30 | const lTags = loggerTagsFactory('transcoding') | ||
| 31 | |||
| 32 | async function processVideoTranscoding (job: Job) { | ||
| 33 | const payload = job.data as VideoTranscodingPayload | ||
| 34 | logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID)) | ||
| 35 | |||
| 36 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
| 37 | // No video, maybe deleted? | ||
| 38 | if (!video) { | ||
| 39 | logger.info('Do not process job %d, video does not exist.', job.id, lTags(payload.videoUUID)) | ||
| 40 | return undefined | ||
| 41 | } | ||
| 42 | |||
| 43 | const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) | ||
| 44 | |||
| 45 | const handler = handlers[payload.type] | ||
| 46 | |||
| 47 | if (!handler) { | ||
| 48 | await moveToFailedTranscodingState(video) | ||
| 49 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
| 50 | |||
| 51 | throw new Error('Cannot find transcoding handler for ' + payload.type) | ||
| 52 | } | ||
| 53 | |||
| 54 | try { | ||
| 55 | await handler(job, payload, video, user) | ||
| 56 | } catch (error) { | ||
| 57 | await moveToFailedTranscodingState(video) | ||
| 58 | |||
| 59 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
| 60 | |||
| 61 | throw error | ||
| 62 | } | ||
| 63 | |||
| 64 | return video | ||
| 65 | } | ||
| 66 | |||
| 67 | // --------------------------------------------------------------------------- | ||
| 68 | |||
| 69 | export { | ||
| 70 | processVideoTranscoding | ||
| 71 | } | ||
| 72 | |||
| 73 | // --------------------------------------------------------------------------- | ||
| 74 | // Job handlers | ||
| 75 | // --------------------------------------------------------------------------- | ||
| 76 | |||
| 77 | async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { | ||
| 78 | logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) | ||
| 79 | |||
| 80 | await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) | ||
| 81 | |||
| 82 | logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) | ||
| 83 | |||
| 84 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) | ||
| 85 | } | ||
| 86 | |||
| 87 | async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { | ||
| 88 | logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) | ||
| 89 | |||
| 90 | await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) | ||
| 91 | |||
| 92 | logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) | ||
| 93 | |||
| 94 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) | ||
| 95 | } | ||
| 96 | |||
| 97 | // --------------------------------------------------------------------------- | ||
| 98 | |||
| 99 | async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) { | ||
| 100 | logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) | ||
| 101 | |||
| 102 | await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) | ||
| 103 | |||
| 104 | logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) | ||
| 105 | |||
| 106 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) | ||
| 107 | } | ||
| 108 | |||
| 109 | // --------------------------------------------------------------------------- | ||
| 110 | |||
| 111 | async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFullLight) { | ||
| 112 | logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload }) | ||
| 113 | |||
| 114 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) | ||
| 115 | let video: MVideoFullLight | ||
| 116 | |||
| 117 | try { | ||
| 118 | video = await VideoModel.loadFull(videoArg.uuid) | ||
| 119 | |||
| 120 | const videoFileInput = payload.copyCodecs | ||
| 121 | ? video.getWebVideoFile(payload.resolution) | ||
| 122 | : video.getMaxQualityFile() | ||
| 123 | |||
| 124 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() | ||
| 125 | |||
| 126 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { | ||
| 127 | return generateHlsPlaylistResolution({ | ||
| 128 | video, | ||
| 129 | videoInputPath, | ||
| 130 | inputFileMutexReleaser, | ||
| 131 | resolution: payload.resolution, | ||
| 132 | fps: payload.fps, | ||
| 133 | copyCodecs: payload.copyCodecs, | ||
| 134 | job | ||
| 135 | }) | ||
| 136 | }) | ||
| 137 | } finally { | ||
| 138 | inputFileMutexReleaser() | ||
| 139 | } | ||
| 140 | |||
| 141 | logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) | ||
| 142 | |||
| 143 | if (payload.deleteWebVideoFiles === true) { | ||
| 144 | logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) | ||
| 145 | |||
| 146 | await removeAllWebVideoFiles(video) | ||
| 147 | } | ||
| 148 | |||
| 149 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) | ||
| 150 | } | ||
diff --git a/server/lib/job-queue/handlers/video-views-stats.ts b/server/lib/job-queue/handlers/video-views-stats.ts deleted file mode 100644 index c9aa218e5..000000000 --- a/server/lib/job-queue/handlers/video-views-stats.ts +++ /dev/null | |||
| @@ -1,57 +0,0 @@ | |||
| 1 | import { VideoViewModel } from '@server/models/view/video-view' | ||
| 2 | import { isTestOrDevInstance } from '../../../helpers/core-utils' | ||
| 3 | import { logger } from '../../../helpers/logger' | ||
| 4 | import { VideoModel } from '../../../models/video/video' | ||
| 5 | import { Redis } from '../../redis' | ||
| 6 | |||
| 7 | async function processVideosViewsStats () { | ||
| 8 | const lastHour = new Date() | ||
| 9 | |||
| 10 | // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour | ||
| 11 | if (!isTestOrDevInstance()) lastHour.setHours(lastHour.getHours() - 1) | ||
| 12 | |||
| 13 | const hour = lastHour.getHours() | ||
| 14 | const startDate = lastHour.setMinutes(0, 0, 0) | ||
| 15 | const endDate = lastHour.setMinutes(59, 59, 999) | ||
| 16 | |||
| 17 | const videoIds = await Redis.Instance.listVideosViewedForStats(hour) | ||
| 18 | if (videoIds.length === 0) return | ||
| 19 | |||
| 20 | logger.info('Processing videos views stats in job for hour %d.', hour) | ||
| 21 | |||
| 22 | for (const videoId of videoIds) { | ||
| 23 | try { | ||
| 24 | const views = await Redis.Instance.getVideoViewsStats(videoId, hour) | ||
| 25 | await Redis.Instance.deleteVideoViewsStats(videoId, hour) | ||
| 26 | |||
| 27 | if (views) { | ||
| 28 | logger.debug('Adding %d views to video %d stats in hour %d.', views, videoId, hour) | ||
| 29 | |||
| 30 | try { | ||
| 31 | const video = await VideoModel.load(videoId) | ||
| 32 | if (!video) { | ||
| 33 | logger.debug('Video %d does not exist anymore, skipping videos view stats.', videoId) | ||
| 34 | continue | ||
| 35 | } | ||
| 36 | |||
| 37 | await VideoViewModel.create({ | ||
| 38 | startDate: new Date(startDate), | ||
| 39 | endDate: new Date(endDate), | ||
| 40 | views, | ||
| 41 | videoId | ||
| 42 | }) | ||
| 43 | } catch (err) { | ||
| 44 | logger.error('Cannot create video views stats for video %d in hour %d.', videoId, hour, { err }) | ||
| 45 | } | ||
| 46 | } | ||
| 47 | } catch (err) { | ||
| 48 | logger.error('Cannot update video views stats of video %d in hour %d.', videoId, hour, { err }) | ||
| 49 | } | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | // --------------------------------------------------------------------------- | ||
| 54 | |||
| 55 | export { | ||
| 56 | processVideosViewsStats | ||
| 57 | } | ||
diff --git a/server/lib/job-queue/index.ts b/server/lib/job-queue/index.ts deleted file mode 100644 index 57231e649..000000000 --- a/server/lib/job-queue/index.ts +++ /dev/null | |||
| @@ -1 +0,0 @@ | |||
| 1 | export * from './job-queue' | ||
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts deleted file mode 100644 index 177bca285..000000000 --- a/server/lib/job-queue/job-queue.ts +++ /dev/null | |||
| @@ -1,537 +0,0 @@ | |||
| 1 | import { | ||
| 2 | FlowJob, | ||
| 3 | FlowProducer, | ||
| 4 | Job, | ||
| 5 | JobsOptions, | ||
| 6 | Queue, | ||
| 7 | QueueEvents, | ||
| 8 | QueueEventsOptions, | ||
| 9 | QueueOptions, | ||
| 10 | Worker, | ||
| 11 | WorkerOptions | ||
| 12 | } from 'bullmq' | ||
| 13 | import { parseDurationToMs } from '@server/helpers/core-utils' | ||
| 14 | import { jobStates } from '@server/helpers/custom-validators/jobs' | ||
| 15 | import { CONFIG } from '@server/initializers/config' | ||
| 16 | import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy' | ||
| 17 | import { pick, timeoutPromise } from '@shared/core-utils' | ||
| 18 | import { | ||
| 19 | ActivitypubFollowPayload, | ||
| 20 | ActivitypubHttpBroadcastPayload, | ||
| 21 | ActivitypubHttpFetcherPayload, | ||
| 22 | ActivitypubHttpUnicastPayload, | ||
| 23 | ActorKeysPayload, | ||
| 24 | AfterVideoChannelImportPayload, | ||
| 25 | DeleteResumableUploadMetaFilePayload, | ||
| 26 | EmailPayload, | ||
| 27 | FederateVideoPayload, | ||
| 28 | GenerateStoryboardPayload, | ||
| 29 | JobState, | ||
| 30 | JobType, | ||
| 31 | ManageVideoTorrentPayload, | ||
| 32 | MoveObjectStoragePayload, | ||
| 33 | NotifyPayload, | ||
| 34 | RefreshPayload, | ||
| 35 | TranscodingJobBuilderPayload, | ||
| 36 | VideoChannelImportPayload, | ||
| 37 | VideoFileImportPayload, | ||
| 38 | VideoImportPayload, | ||
| 39 | VideoLiveEndingPayload, | ||
| 40 | VideoRedundancyPayload, | ||
| 41 | VideoStudioEditionPayload, | ||
| 42 | VideoTranscodingPayload | ||
| 43 | } from '../../../shared/models' | ||
| 44 | import { logger } from '../../helpers/logger' | ||
| 45 | import { JOB_ATTEMPTS, JOB_CONCURRENCY, JOB_REMOVAL_OPTIONS, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' | ||
| 46 | import { Hooks } from '../plugins/hooks' | ||
| 47 | import { Redis } from '../redis' | ||
| 48 | import { processActivityPubCleaner } from './handlers/activitypub-cleaner' | ||
| 49 | import { processActivityPubFollow } from './handlers/activitypub-follow' | ||
| 50 | import { processActivityPubHttpSequentialBroadcast, processActivityPubParallelHttpBroadcast } from './handlers/activitypub-http-broadcast' | ||
| 51 | import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | ||
| 52 | import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | ||
| 53 | import { refreshAPObject } from './handlers/activitypub-refresher' | ||
| 54 | import { processActorKeys } from './handlers/actor-keys' | ||
| 55 | import { processAfterVideoChannelImport } from './handlers/after-video-channel-import' | ||
| 56 | import { processEmail } from './handlers/email' | ||
| 57 | import { processFederateVideo } from './handlers/federate-video' | ||
| 58 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' | ||
| 59 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' | ||
| 60 | import { processNotify } from './handlers/notify' | ||
| 61 | import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder' | ||
| 62 | import { processVideoChannelImport } from './handlers/video-channel-import' | ||
| 63 | import { processVideoFileImport } from './handlers/video-file-import' | ||
| 64 | import { processVideoImport } from './handlers/video-import' | ||
| 65 | import { processVideoLiveEnding } from './handlers/video-live-ending' | ||
| 66 | import { processVideoStudioEdition } from './handlers/video-studio-edition' | ||
| 67 | import { processVideoTranscoding } from './handlers/video-transcoding' | ||
| 68 | import { processVideosViewsStats } from './handlers/video-views-stats' | ||
| 69 | import { processGenerateStoryboard } from './handlers/generate-storyboard' | ||
| 70 | |||
| 71 | export type CreateJobArgument = | ||
| 72 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | ||
| 73 | { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | | ||
| 74 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | ||
| 75 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | ||
| 76 | { type: 'activitypub-cleaner', payload: {} } | | ||
| 77 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | | ||
| 78 | { type: 'video-file-import', payload: VideoFileImportPayload } | | ||
| 79 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | | ||
| 80 | { type: 'email', payload: EmailPayload } | | ||
| 81 | { type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } | | ||
| 82 | { type: 'video-import', payload: VideoImportPayload } | | ||
| 83 | { type: 'activitypub-refresher', payload: RefreshPayload } | | ||
| 84 | { type: 'videos-views-stats', payload: {} } | | ||
| 85 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | | ||
| 86 | { type: 'actor-keys', payload: ActorKeysPayload } | | ||
| 87 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | | ||
| 88 | { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | | ||
| 89 | { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | | ||
| 90 | { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | | ||
| 91 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | | ||
| 92 | { type: 'video-channel-import', payload: VideoChannelImportPayload } | | ||
| 93 | { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | | ||
| 94 | { type: 'notify', payload: NotifyPayload } | | ||
| 95 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | | ||
| 96 | { type: 'federate-video', payload: FederateVideoPayload } | | ||
| 97 | { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } | ||
| 98 | |||
| 99 | export type CreateJobOptions = { | ||
| 100 | delay?: number | ||
| 101 | priority?: number | ||
| 102 | failParentOnFailure?: boolean | ||
| 103 | } | ||
| 104 | |||
| 105 | const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { | ||
| 106 | 'activitypub-cleaner': processActivityPubCleaner, | ||
| 107 | 'activitypub-follow': processActivityPubFollow, | ||
| 108 | 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, | ||
| 109 | 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, | ||
| 110 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | ||
| 111 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | ||
| 112 | 'activitypub-refresher': refreshAPObject, | ||
| 113 | 'actor-keys': processActorKeys, | ||
| 114 | 'after-video-channel-import': processAfterVideoChannelImport, | ||
| 115 | 'email': processEmail, | ||
| 116 | 'federate-video': processFederateVideo, | ||
| 117 | 'transcoding-job-builder': processTranscodingJobBuilder, | ||
| 118 | 'manage-video-torrent': processManageVideoTorrent, | ||
| 119 | 'move-to-object-storage': processMoveToObjectStorage, | ||
| 120 | 'notify': processNotify, | ||
| 121 | 'video-channel-import': processVideoChannelImport, | ||
| 122 | 'video-file-import': processVideoFileImport, | ||
| 123 | 'video-import': processVideoImport, | ||
| 124 | 'video-live-ending': processVideoLiveEnding, | ||
| 125 | 'video-redundancy': processVideoRedundancy, | ||
| 126 | 'video-studio-edition': processVideoStudioEdition, | ||
| 127 | 'video-transcoding': processVideoTranscoding, | ||
| 128 | 'videos-views-stats': processVideosViewsStats, | ||
| 129 | 'generate-video-storyboard': processGenerateStoryboard | ||
| 130 | } | ||
| 131 | |||
| 132 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { | ||
| 133 | 'move-to-object-storage': onMoveToObjectStorageFailure | ||
| 134 | } | ||
| 135 | |||
| 136 | const jobTypes: JobType[] = [ | ||
| 137 | 'activitypub-cleaner', | ||
| 138 | 'activitypub-follow', | ||
| 139 | 'activitypub-http-broadcast-parallel', | ||
| 140 | 'activitypub-http-broadcast', | ||
| 141 | 'activitypub-http-fetcher', | ||
| 142 | 'activitypub-http-unicast', | ||
| 143 | 'activitypub-refresher', | ||
| 144 | 'actor-keys', | ||
| 145 | 'after-video-channel-import', | ||
| 146 | 'email', | ||
| 147 | 'federate-video', | ||
| 148 | 'generate-video-storyboard', | ||
| 149 | 'manage-video-torrent', | ||
| 150 | 'move-to-object-storage', | ||
| 151 | 'notify', | ||
| 152 | 'transcoding-job-builder', | ||
| 153 | 'video-channel-import', | ||
| 154 | 'video-file-import', | ||
| 155 | 'video-import', | ||
| 156 | 'video-live-ending', | ||
| 157 | 'video-redundancy', | ||
| 158 | 'video-studio-edition', | ||
| 159 | 'video-transcoding', | ||
| 160 | 'videos-views-stats' | ||
| 161 | ] | ||
| 162 | |||
| 163 | const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ]) | ||
| 164 | |||
| 165 | class JobQueue { | ||
| 166 | |||
| 167 | private static instance: JobQueue | ||
| 168 | |||
| 169 | private workers: { [id in JobType]?: Worker } = {} | ||
| 170 | private queues: { [id in JobType]?: Queue } = {} | ||
| 171 | private queueEvents: { [id in JobType]?: QueueEvents } = {} | ||
| 172 | |||
| 173 | private flowProducer: FlowProducer | ||
| 174 | |||
| 175 | private initialized = false | ||
| 176 | private jobRedisPrefix: string | ||
| 177 | |||
| 178 | private constructor () { | ||
| 179 | } | ||
| 180 | |||
| 181 | init () { | ||
| 182 | // Already initialized | ||
| 183 | if (this.initialized === true) return | ||
| 184 | this.initialized = true | ||
| 185 | |||
| 186 | this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST | ||
| 187 | |||
| 188 | for (const handlerName of Object.keys(handlers)) { | ||
| 189 | this.buildWorker(handlerName) | ||
| 190 | this.buildQueue(handlerName) | ||
| 191 | this.buildQueueEvent(handlerName) | ||
| 192 | } | ||
| 193 | |||
| 194 | this.flowProducer = new FlowProducer({ | ||
| 195 | connection: Redis.getRedisClientOptions('FlowProducer'), | ||
| 196 | prefix: this.jobRedisPrefix | ||
| 197 | }) | ||
| 198 | this.flowProducer.on('error', err => { logger.error('Error in flow producer', { err }) }) | ||
| 199 | |||
| 200 | this.addRepeatableJobs() | ||
| 201 | } | ||
| 202 | |||
| 203 | private buildWorker (handlerName: JobType) { | ||
| 204 | const workerOptions: WorkerOptions = { | ||
| 205 | autorun: false, | ||
| 206 | concurrency: this.getJobConcurrency(handlerName), | ||
| 207 | prefix: this.jobRedisPrefix, | ||
| 208 | connection: Redis.getRedisClientOptions('Worker'), | ||
| 209 | maxStalledCount: 10 | ||
| 210 | } | ||
| 211 | |||
| 212 | const handler = function (job: Job) { | ||
| 213 | const timeout = JOB_TTL[handlerName] | ||
| 214 | const p = handlers[handlerName](job) | ||
| 215 | |||
| 216 | if (!timeout) return p | ||
| 217 | |||
| 218 | return timeoutPromise(p, timeout) | ||
| 219 | } | ||
| 220 | |||
| 221 | const processor = async (jobArg: Job<any>) => { | ||
| 222 | const job = await Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName }) | ||
| 223 | |||
| 224 | return Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result') | ||
| 225 | } | ||
| 226 | |||
| 227 | const worker = new Worker(handlerName, processor, workerOptions) | ||
| 228 | |||
| 229 | worker.on('failed', (job, err) => { | ||
| 230 | const logLevel = silentFailure.has(handlerName) | ||
| 231 | ? 'debug' | ||
| 232 | : 'error' | ||
| 233 | |||
| 234 | logger.log(logLevel, 'Cannot execute job %s in queue %s.', job.id, handlerName, { payload: job.data, err }) | ||
| 235 | |||
| 236 | if (errorHandlers[job.name]) { | ||
| 237 | errorHandlers[job.name](job, err) | ||
| 238 | .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err })) | ||
| 239 | } | ||
| 240 | }) | ||
| 241 | |||
| 242 | worker.on('error', err => { logger.error('Error in job worker %s.', handlerName, { err }) }) | ||
| 243 | |||
| 244 | this.workers[handlerName] = worker | ||
| 245 | } | ||
| 246 | |||
| 247 | private buildQueue (handlerName: JobType) { | ||
| 248 | const queueOptions: QueueOptions = { | ||
| 249 | connection: Redis.getRedisClientOptions('Queue'), | ||
| 250 | prefix: this.jobRedisPrefix | ||
| 251 | } | ||
| 252 | |||
| 253 | const queue = new Queue(handlerName, queueOptions) | ||
| 254 | queue.on('error', err => { logger.error('Error in job queue %s.', handlerName, { err }) }) | ||
| 255 | |||
| 256 | this.queues[handlerName] = queue | ||
| 257 | } | ||
| 258 | |||
| 259 | private buildQueueEvent (handlerName: JobType) { | ||
| 260 | const queueEventsOptions: QueueEventsOptions = { | ||
| 261 | autorun: false, | ||
| 262 | connection: Redis.getRedisClientOptions('QueueEvent'), | ||
| 263 | prefix: this.jobRedisPrefix | ||
| 264 | } | ||
| 265 | |||
| 266 | const queueEvents = new QueueEvents(handlerName, queueEventsOptions) | ||
| 267 | queueEvents.on('error', err => { logger.error('Error in job queue events %s.', handlerName, { err }) }) | ||
| 268 | |||
| 269 | this.queueEvents[handlerName] = queueEvents | ||
| 270 | } | ||
| 271 | |||
| 272 | // --------------------------------------------------------------------------- | ||
| 273 | |||
| 274 | async terminate () { | ||
| 275 | const promises = Object.keys(this.workers) | ||
| 276 | .map(handlerName => { | ||
| 277 | const worker: Worker = this.workers[handlerName] | ||
| 278 | const queue: Queue = this.queues[handlerName] | ||
| 279 | const queueEvent: QueueEvents = this.queueEvents[handlerName] | ||
| 280 | |||
| 281 | return Promise.all([ | ||
| 282 | worker.close(false), | ||
| 283 | queue.close(), | ||
| 284 | queueEvent.close() | ||
| 285 | ]) | ||
| 286 | }) | ||
| 287 | |||
| 288 | return Promise.all(promises) | ||
| 289 | } | ||
| 290 | |||
| 291 | start () { | ||
| 292 | const promises = Object.keys(this.workers) | ||
| 293 | .map(handlerName => { | ||
| 294 | const worker: Worker = this.workers[handlerName] | ||
| 295 | const queueEvent: QueueEvents = this.queueEvents[handlerName] | ||
| 296 | |||
| 297 | return Promise.all([ | ||
| 298 | worker.run(), | ||
| 299 | queueEvent.run() | ||
| 300 | ]) | ||
| 301 | }) | ||
| 302 | |||
| 303 | return Promise.all(promises) | ||
| 304 | } | ||
| 305 | |||
| 306 | async pause () { | ||
| 307 | for (const handlerName of Object.keys(this.workers)) { | ||
| 308 | const worker: Worker = this.workers[handlerName] | ||
| 309 | |||
| 310 | await worker.pause() | ||
| 311 | } | ||
| 312 | } | ||
| 313 | |||
| 314 | resume () { | ||
| 315 | for (const handlerName of Object.keys(this.workers)) { | ||
| 316 | const worker: Worker = this.workers[handlerName] | ||
| 317 | |||
| 318 | worker.resume() | ||
| 319 | } | ||
| 320 | } | ||
| 321 | |||
| 322 | // --------------------------------------------------------------------------- | ||
| 323 | |||
| 324 | createJobAsync (options: CreateJobArgument & CreateJobOptions): void { | ||
| 325 | this.createJob(options) | ||
| 326 | .catch(err => logger.error('Cannot create job.', { err, options })) | ||
| 327 | } | ||
| 328 | |||
| 329 | createJob (options: CreateJobArgument & CreateJobOptions) { | ||
| 330 | const queue: Queue = this.queues[options.type] | ||
| 331 | if (queue === undefined) { | ||
| 332 | logger.error('Unknown queue %s: cannot create job.', options.type) | ||
| 333 | return | ||
| 334 | } | ||
| 335 | |||
| 336 | const jobOptions = this.buildJobOptions(options.type as JobType, pick(options, [ 'priority', 'delay' ])) | ||
| 337 | |||
| 338 | return queue.add('job', options.payload, jobOptions) | ||
| 339 | } | ||
| 340 | |||
| 341 | createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { | ||
| 342 | let lastJob: FlowJob | ||
| 343 | |||
| 344 | for (const job of jobs) { | ||
| 345 | if (!job) continue | ||
| 346 | |||
| 347 | lastJob = { | ||
| 348 | ...this.buildJobFlowOption(job), | ||
| 349 | |||
| 350 | children: lastJob | ||
| 351 | ? [ lastJob ] | ||
| 352 | : [] | ||
| 353 | } | ||
| 354 | } | ||
| 355 | |||
| 356 | return this.flowProducer.add(lastJob) | ||
| 357 | } | ||
| 358 | |||
| 359 | createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { | ||
| 360 | return this.flowProducer.add({ | ||
| 361 | ...this.buildJobFlowOption(parent), | ||
| 362 | |||
| 363 | children: children.map(c => this.buildJobFlowOption(c)) | ||
| 364 | }) | ||
| 365 | } | ||
| 366 | |||
| 367 | private buildJobFlowOption (job: CreateJobArgument & CreateJobOptions): FlowJob { | ||
| 368 | return { | ||
| 369 | name: 'job', | ||
| 370 | data: job.payload, | ||
| 371 | queueName: job.type, | ||
| 372 | opts: { | ||
| 373 | failParentOnFailure: true, | ||
| 374 | |||
| 375 | ...this.buildJobOptions(job.type as JobType, pick(job, [ 'priority', 'delay', 'failParentOnFailure' ])) | ||
| 376 | } | ||
| 377 | } | ||
| 378 | } | ||
| 379 | |||
| 380 | private buildJobOptions (type: JobType, options: CreateJobOptions = {}): JobsOptions { | ||
| 381 | return { | ||
| 382 | backoff: { delay: 60 * 1000, type: 'exponential' }, | ||
| 383 | attempts: JOB_ATTEMPTS[type], | ||
| 384 | priority: options.priority, | ||
| 385 | delay: options.delay, | ||
| 386 | |||
| 387 | ...this.buildJobRemovalOptions(type) | ||
| 388 | } | ||
| 389 | } | ||
| 390 | |||
| 391 | // --------------------------------------------------------------------------- | ||
| 392 | |||
| 393 | async listForApi (options: { | ||
| 394 | state?: JobState | ||
| 395 | start: number | ||
| 396 | count: number | ||
| 397 | asc?: boolean | ||
| 398 | jobType: JobType | ||
| 399 | }): Promise<Job[]> { | ||
| 400 | const { state, start, count, asc, jobType } = options | ||
| 401 | |||
| 402 | const states = this.buildStateFilter(state) | ||
| 403 | const filteredJobTypes = this.buildTypeFilter(jobType) | ||
| 404 | |||
| 405 | let results: Job[] = [] | ||
| 406 | |||
| 407 | for (const jobType of filteredJobTypes) { | ||
| 408 | const queue: Queue = this.queues[jobType] | ||
| 409 | |||
| 410 | if (queue === undefined) { | ||
| 411 | logger.error('Unknown queue %s to list jobs.', jobType) | ||
| 412 | continue | ||
| 413 | } | ||
| 414 | |||
| 415 | const jobs = await queue.getJobs(states, 0, start + count, asc) | ||
| 416 | results = results.concat(jobs) | ||
| 417 | } | ||
| 418 | |||
| 419 | results.sort((j1: any, j2: any) => { | ||
| 420 | if (j1.timestamp < j2.timestamp) return -1 | ||
| 421 | else if (j1.timestamp === j2.timestamp) return 0 | ||
| 422 | |||
| 423 | return 1 | ||
| 424 | }) | ||
| 425 | |||
| 426 | if (asc === false) results.reverse() | ||
| 427 | |||
| 428 | return results.slice(start, start + count) | ||
| 429 | } | ||
| 430 | |||
| 431 | async count (state: JobState, jobType?: JobType): Promise<number> { | ||
| 432 | const states = state ? [ state ] : jobStates | ||
| 433 | const filteredJobTypes = this.buildTypeFilter(jobType) | ||
| 434 | |||
| 435 | let total = 0 | ||
| 436 | |||
| 437 | for (const type of filteredJobTypes) { | ||
| 438 | const queue = this.queues[type] | ||
| 439 | if (queue === undefined) { | ||
| 440 | logger.error('Unknown queue %s to count jobs.', type) | ||
| 441 | continue | ||
| 442 | } | ||
| 443 | |||
| 444 | const counts = await queue.getJobCounts() | ||
| 445 | |||
| 446 | for (const s of states) { | ||
| 447 | total += counts[s] | ||
| 448 | } | ||
| 449 | } | ||
| 450 | |||
| 451 | return total | ||
| 452 | } | ||
| 453 | |||
| 454 | private buildStateFilter (state?: JobState) { | ||
| 455 | if (!state) return jobStates | ||
| 456 | |||
| 457 | const states = [ state ] | ||
| 458 | |||
| 459 | // Include parent if filtering on waiting | ||
| 460 | if (state === 'waiting') states.push('waiting-children') | ||
| 461 | |||
| 462 | return states | ||
| 463 | } | ||
| 464 | |||
| 465 | private buildTypeFilter (jobType?: JobType) { | ||
| 466 | if (!jobType) return jobTypes | ||
| 467 | |||
| 468 | return jobTypes.filter(t => t === jobType) | ||
| 469 | } | ||
| 470 | |||
| 471 | async getStats () { | ||
| 472 | const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() })) | ||
| 473 | |||
| 474 | return Promise.all(promises) | ||
| 475 | } | ||
| 476 | |||
| 477 | // --------------------------------------------------------------------------- | ||
| 478 | |||
| 479 | async removeOldJobs () { | ||
| 480 | for (const key of Object.keys(this.queues)) { | ||
| 481 | const queue: Queue = this.queues[key] | ||
| 482 | await queue.clean(parseDurationToMs('7 days'), 1000, 'completed') | ||
| 483 | await queue.clean(parseDurationToMs('7 days'), 1000, 'failed') | ||
| 484 | } | ||
| 485 | } | ||
| 486 | |||
| 487 | private addRepeatableJobs () { | ||
| 488 | this.queues['videos-views-stats'].add('job', {}, { | ||
| 489 | repeat: REPEAT_JOBS['videos-views-stats'], | ||
| 490 | |||
| 491 | ...this.buildJobRemovalOptions('videos-views-stats') | ||
| 492 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) | ||
| 493 | |||
| 494 | if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) { | ||
| 495 | this.queues['activitypub-cleaner'].add('job', {}, { | ||
| 496 | repeat: REPEAT_JOBS['activitypub-cleaner'], | ||
| 497 | |||
| 498 | ...this.buildJobRemovalOptions('activitypub-cleaner') | ||
| 499 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) | ||
| 500 | } | ||
| 501 | } | ||
| 502 | |||
| 503 | private getJobConcurrency (jobType: JobType) { | ||
| 504 | if (jobType === 'video-transcoding') return CONFIG.TRANSCODING.CONCURRENCY | ||
| 505 | if (jobType === 'video-import') return CONFIG.IMPORT.VIDEOS.CONCURRENCY | ||
| 506 | |||
| 507 | return JOB_CONCURRENCY[jobType] | ||
| 508 | } | ||
| 509 | |||
| 510 | private buildJobRemovalOptions (queueName: string) { | ||
| 511 | return { | ||
| 512 | removeOnComplete: { | ||
| 513 | // Wants seconds | ||
| 514 | age: (JOB_REMOVAL_OPTIONS.SUCCESS[queueName] || JOB_REMOVAL_OPTIONS.SUCCESS.DEFAULT) / 1000, | ||
| 515 | |||
| 516 | count: JOB_REMOVAL_OPTIONS.COUNT | ||
| 517 | }, | ||
| 518 | removeOnFail: { | ||
| 519 | // Wants seconds | ||
| 520 | age: (JOB_REMOVAL_OPTIONS.FAILURE[queueName] || JOB_REMOVAL_OPTIONS.FAILURE.DEFAULT) / 1000, | ||
| 521 | |||
| 522 | count: JOB_REMOVAL_OPTIONS.COUNT / 1000 | ||
| 523 | } | ||
| 524 | } | ||
| 525 | } | ||
| 526 | |||
| 527 | static get Instance () { | ||
| 528 | return this.instance || (this.instance = new this()) | ||
| 529 | } | ||
| 530 | } | ||
| 531 | |||
| 532 | // --------------------------------------------------------------------------- | ||
| 533 | |||
| 534 | export { | ||
| 535 | jobTypes, | ||
| 536 | JobQueue | ||
| 537 | } | ||
diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts deleted file mode 100644 index 8b46800da..000000000 --- a/server/lib/live/index.ts +++ /dev/null | |||
| @@ -1,4 +0,0 @@ | |||
| 1 | export * from './live-manager' | ||
| 2 | export * from './live-quota-store' | ||
| 3 | export * from './live-segment-sha-store' | ||
| 4 | export * from './live-utils' | ||
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts deleted file mode 100644 index acb7af274..000000000 --- a/server/lib/live/live-manager.ts +++ /dev/null | |||
| @@ -1,552 +0,0 @@ | |||
| 1 | import { readdir, readFile } from 'fs-extra' | ||
| 2 | import { createServer, Server } from 'net' | ||
| 3 | import { join } from 'path' | ||
| 4 | import { createServer as createServerTLS, Server as ServerTLS } from 'tls' | ||
| 5 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 6 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | ||
| 7 | import { VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants' | ||
| 8 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 9 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
| 10 | import { UserModel } from '@server/models/user/user' | ||
| 11 | import { VideoModel } from '@server/models/video/video' | ||
| 12 | import { VideoLiveModel } from '@server/models/video/video-live' | ||
| 13 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
| 14 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' | ||
| 15 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
| 16 | import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models' | ||
| 17 | import { pick, wait } from '@shared/core-utils' | ||
| 18 | import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg' | ||
| 19 | import { LiveVideoError, VideoState } from '@shared/models' | ||
| 20 | import { federateVideoIfNeeded } from '../activitypub/videos' | ||
| 21 | import { JobQueue } from '../job-queue' | ||
| 22 | import { getLiveReplayBaseDirectory } from '../paths' | ||
| 23 | import { PeerTubeSocket } from '../peertube-socket' | ||
| 24 | import { Hooks } from '../plugins/hooks' | ||
| 25 | import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions' | ||
| 26 | import { LiveQuotaStore } from './live-quota-store' | ||
| 27 | import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils' | ||
| 28 | import { MuxingSession } from './shared' | ||
| 29 | |||
| 30 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') | ||
| 31 | const context = require('node-media-server/src/node_core_ctx') | ||
| 32 | const nodeMediaServerLogger = require('node-media-server/src/node_core_logger') | ||
| 33 | |||
| 34 | // Disable node media server logs | ||
| 35 | nodeMediaServerLogger.setLogType(0) | ||
| 36 | |||
| 37 | const config = { | ||
| 38 | rtmp: { | ||
| 39 | port: CONFIG.LIVE.RTMP.PORT, | ||
| 40 | chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, | ||
| 41 | gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, | ||
| 42 | ping: VIDEO_LIVE.RTMP.PING, | ||
| 43 | ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT | ||
| 44 | } | ||
| 45 | } | ||
| 46 | |||
| 47 | const lTags = loggerTagsFactory('live') | ||
| 48 | |||
| 49 | class LiveManager { | ||
| 50 | |||
| 51 | private static instance: LiveManager | ||
| 52 | |||
| 53 | private readonly muxingSessions = new Map<string, MuxingSession>() | ||
| 54 | private readonly videoSessions = new Map<string, string>() | ||
| 55 | |||
| 56 | private rtmpServer: Server | ||
| 57 | private rtmpsServer: ServerTLS | ||
| 58 | |||
| 59 | private running = false | ||
| 60 | |||
| 61 | private constructor () { | ||
| 62 | } | ||
| 63 | |||
| 64 | init () { | ||
| 65 | const events = this.getContext().nodeEvent | ||
| 66 | events.on('postPublish', (sessionId: string, streamPath: string) => { | ||
| 67 | logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) | ||
| 68 | |||
| 69 | const splittedPath = streamPath.split('/') | ||
| 70 | if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { | ||
| 71 | logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) | ||
| 72 | return this.abortSession(sessionId) | ||
| 73 | } | ||
| 74 | |||
| 75 | const session = this.getContext().sessions.get(sessionId) | ||
| 76 | const inputLocalUrl = session.inputOriginLocalUrl + streamPath | ||
| 77 | const inputPublicUrl = session.inputOriginPublicUrl + streamPath | ||
| 78 | |||
| 79 | this.handleSession({ sessionId, inputPublicUrl, inputLocalUrl, streamKey: splittedPath[2] }) | ||
| 80 | .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) | ||
| 81 | }) | ||
| 82 | |||
| 83 | events.on('donePublish', sessionId => { | ||
| 84 | logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) | ||
| 85 | |||
| 86 | // Force session aborting, so we kill ffmpeg even if it still has data to process (slow CPU) | ||
| 87 | setTimeout(() => this.abortSession(sessionId), 2000) | ||
| 88 | }) | ||
| 89 | |||
| 90 | registerConfigChangedHandler(() => { | ||
| 91 | if (!this.running && CONFIG.LIVE.ENABLED === true) { | ||
| 92 | this.run().catch(err => logger.error('Cannot run live server.', { err })) | ||
| 93 | return | ||
| 94 | } | ||
| 95 | |||
| 96 | if (this.running && CONFIG.LIVE.ENABLED === false) { | ||
| 97 | this.stop() | ||
| 98 | } | ||
| 99 | }) | ||
| 100 | |||
| 101 | // Cleanup broken lives, that were terminated by a server restart for example | ||
| 102 | this.handleBrokenLives() | ||
| 103 | .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) | ||
| 104 | } | ||
| 105 | |||
| 106 | async run () { | ||
| 107 | this.running = true | ||
| 108 | |||
| 109 | if (CONFIG.LIVE.RTMP.ENABLED) { | ||
| 110 | logger.info('Running RTMP server on port %d', CONFIG.LIVE.RTMP.PORT, lTags()) | ||
| 111 | |||
| 112 | this.rtmpServer = createServer(socket => { | ||
| 113 | const session = new NodeRtmpSession(config, socket) | ||
| 114 | |||
| 115 | session.inputOriginLocalUrl = 'rtmp://127.0.0.1:' + CONFIG.LIVE.RTMP.PORT | ||
| 116 | session.inputOriginPublicUrl = WEBSERVER.RTMP_URL | ||
| 117 | session.run() | ||
| 118 | }) | ||
| 119 | |||
| 120 | this.rtmpServer.on('error', err => { | ||
| 121 | logger.error('Cannot run RTMP server.', { err, ...lTags() }) | ||
| 122 | }) | ||
| 123 | |||
| 124 | this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT, CONFIG.LIVE.RTMP.HOSTNAME) | ||
| 125 | } | ||
| 126 | |||
| 127 | if (CONFIG.LIVE.RTMPS.ENABLED) { | ||
| 128 | logger.info('Running RTMPS server on port %d', CONFIG.LIVE.RTMPS.PORT, lTags()) | ||
| 129 | |||
| 130 | const [ key, cert ] = await Promise.all([ | ||
| 131 | readFile(CONFIG.LIVE.RTMPS.KEY_FILE), | ||
| 132 | readFile(CONFIG.LIVE.RTMPS.CERT_FILE) | ||
| 133 | ]) | ||
| 134 | const serverOptions = { key, cert } | ||
| 135 | |||
| 136 | this.rtmpsServer = createServerTLS(serverOptions, socket => { | ||
| 137 | const session = new NodeRtmpSession(config, socket) | ||
| 138 | |||
| 139 | session.inputOriginLocalUrl = 'rtmps://127.0.0.1:' + CONFIG.LIVE.RTMPS.PORT | ||
| 140 | session.inputOriginPublicUrl = WEBSERVER.RTMPS_URL | ||
| 141 | session.run() | ||
| 142 | }) | ||
| 143 | |||
| 144 | this.rtmpsServer.on('error', err => { | ||
| 145 | logger.error('Cannot run RTMPS server.', { err, ...lTags() }) | ||
| 146 | }) | ||
| 147 | |||
| 148 | this.rtmpsServer.listen(CONFIG.LIVE.RTMPS.PORT, CONFIG.LIVE.RTMPS.HOSTNAME) | ||
| 149 | } | ||
| 150 | } | ||
| 151 | |||
| 152 | stop () { | ||
| 153 | this.running = false | ||
| 154 | |||
| 155 | if (this.rtmpServer) { | ||
| 156 | logger.info('Stopping RTMP server.', lTags()) | ||
| 157 | |||
| 158 | this.rtmpServer.close() | ||
| 159 | this.rtmpServer = undefined | ||
| 160 | } | ||
| 161 | |||
| 162 | if (this.rtmpsServer) { | ||
| 163 | logger.info('Stopping RTMPS server.', lTags()) | ||
| 164 | |||
| 165 | this.rtmpsServer.close() | ||
| 166 | this.rtmpsServer = undefined | ||
| 167 | } | ||
| 168 | |||
| 169 | // Sessions is an object | ||
| 170 | this.getContext().sessions.forEach((session: any) => { | ||
| 171 | if (session instanceof NodeRtmpSession) { | ||
| 172 | session.stop() | ||
| 173 | } | ||
| 174 | }) | ||
| 175 | } | ||
| 176 | |||
| 177 | isRunning () { | ||
| 178 | return !!this.rtmpServer | ||
| 179 | } | ||
| 180 | |||
| 181 | hasSession (sessionId: string) { | ||
| 182 | return this.getContext().sessions.has(sessionId) | ||
| 183 | } | ||
| 184 | |||
| 185 | stopSessionOf (videoUUID: string, error: LiveVideoError | null) { | ||
| 186 | const sessionId = this.videoSessions.get(videoUUID) | ||
| 187 | if (!sessionId) { | ||
| 188 | logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID)) | ||
| 189 | return | ||
| 190 | } | ||
| 191 | |||
| 192 | logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) }) | ||
| 193 | |||
| 194 | this.saveEndingSession(videoUUID, error) | ||
| 195 | .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) })) | ||
| 196 | |||
| 197 | this.videoSessions.delete(videoUUID) | ||
| 198 | this.abortSession(sessionId) | ||
| 199 | } | ||
| 200 | |||
| 201 | private getContext () { | ||
| 202 | return context | ||
| 203 | } | ||
| 204 | |||
| 205 | private abortSession (sessionId: string) { | ||
| 206 | const session = this.getContext().sessions.get(sessionId) | ||
| 207 | if (session) { | ||
| 208 | session.stop() | ||
| 209 | this.getContext().sessions.delete(sessionId) | ||
| 210 | } | ||
| 211 | |||
| 212 | const muxingSession = this.muxingSessions.get(sessionId) | ||
| 213 | if (muxingSession) { | ||
| 214 | // Muxing session will fire and event so we correctly cleanup the session | ||
| 215 | muxingSession.abort() | ||
| 216 | |||
| 217 | this.muxingSessions.delete(sessionId) | ||
| 218 | } | ||
| 219 | } | ||
| 220 | |||
| 221 | private async handleSession (options: { | ||
| 222 | sessionId: string | ||
| 223 | inputLocalUrl: string | ||
| 224 | inputPublicUrl: string | ||
| 225 | streamKey: string | ||
| 226 | }) { | ||
| 227 | const { inputLocalUrl, inputPublicUrl, sessionId, streamKey } = options | ||
| 228 | |||
| 229 | const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) | ||
| 230 | if (!videoLive) { | ||
| 231 | logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) | ||
| 232 | return this.abortSession(sessionId) | ||
| 233 | } | ||
| 234 | |||
| 235 | const video = videoLive.Video | ||
| 236 | if (video.isBlacklisted()) { | ||
| 237 | logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) | ||
| 238 | return this.abortSession(sessionId) | ||
| 239 | } | ||
| 240 | |||
| 241 | if (this.videoSessions.has(video.uuid)) { | ||
| 242 | logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid)) | ||
| 243 | return this.abortSession(sessionId) | ||
| 244 | } | ||
| 245 | |||
| 246 | // Cleanup old potential live (could happen with a permanent live) | ||
| 247 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | ||
| 248 | if (oldStreamingPlaylist) { | ||
| 249 | if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) | ||
| 250 | |||
| 251 | await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) | ||
| 252 | } | ||
| 253 | |||
| 254 | this.videoSessions.set(video.uuid, sessionId) | ||
| 255 | |||
| 256 | const now = Date.now() | ||
| 257 | const probe = await ffprobePromise(inputLocalUrl) | ||
| 258 | |||
| 259 | const [ { resolution, ratio }, fps, bitrate, hasAudio ] = await Promise.all([ | ||
| 260 | getVideoStreamDimensionsInfo(inputLocalUrl, probe), | ||
| 261 | getVideoStreamFPS(inputLocalUrl, probe), | ||
| 262 | getVideoStreamBitrate(inputLocalUrl, probe), | ||
| 263 | hasAudioStream(inputLocalUrl, probe) | ||
| 264 | ]) | ||
| 265 | |||
| 266 | logger.info( | ||
| 267 | '%s probing took %d ms (bitrate: %d, fps: %d, resolution: %d)', | ||
| 268 | inputLocalUrl, Date.now() - now, bitrate, fps, resolution, lTags(sessionId, video.uuid) | ||
| 269 | ) | ||
| 270 | |||
| 271 | const allResolutions = await Hooks.wrapObject( | ||
| 272 | this.buildAllResolutionsToTranscode(resolution, hasAudio), | ||
| 273 | 'filter:transcoding.auto.resolutions-to-transcode.result', | ||
| 274 | { video } | ||
| 275 | ) | ||
| 276 | |||
| 277 | logger.info( | ||
| 278 | 'Handling live video of original resolution %d.', resolution, | ||
| 279 | { allResolutions, ...lTags(sessionId, video.uuid) } | ||
| 280 | ) | ||
| 281 | |||
| 282 | return this.runMuxingSession({ | ||
| 283 | sessionId, | ||
| 284 | videoLive, | ||
| 285 | |||
| 286 | inputLocalUrl, | ||
| 287 | inputPublicUrl, | ||
| 288 | fps, | ||
| 289 | bitrate, | ||
| 290 | ratio, | ||
| 291 | allResolutions, | ||
| 292 | hasAudio | ||
| 293 | }) | ||
| 294 | } | ||
| 295 | |||
| 296 | private async runMuxingSession (options: { | ||
| 297 | sessionId: string | ||
| 298 | videoLive: MVideoLiveVideoWithSetting | ||
| 299 | |||
| 300 | inputLocalUrl: string | ||
| 301 | inputPublicUrl: string | ||
| 302 | |||
| 303 | fps: number | ||
| 304 | bitrate: number | ||
| 305 | ratio: number | ||
| 306 | allResolutions: number[] | ||
| 307 | hasAudio: boolean | ||
| 308 | }) { | ||
| 309 | const { sessionId, videoLive } = options | ||
| 310 | const videoUUID = videoLive.Video.uuid | ||
| 311 | const localLTags = lTags(sessionId, videoUUID) | ||
| 312 | |||
| 313 | const liveSession = await this.saveStartingSession(videoLive) | ||
| 314 | |||
| 315 | const user = await UserModel.loadByLiveId(videoLive.id) | ||
| 316 | LiveQuotaStore.Instance.addNewLive(user.id, sessionId) | ||
| 317 | |||
| 318 | const muxingSession = new MuxingSession({ | ||
| 319 | context: this.getContext(), | ||
| 320 | sessionId, | ||
| 321 | videoLive, | ||
| 322 | user, | ||
| 323 | |||
| 324 | ...pick(options, [ 'inputLocalUrl', 'inputPublicUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) | ||
| 325 | }) | ||
| 326 | |||
| 327 | muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) | ||
| 328 | |||
| 329 | muxingSession.on('bad-socket-health', ({ videoUUID }) => { | ||
| 330 | logger.error( | ||
| 331 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + | ||
| 332 | ' Stopping session of video %s.', videoUUID, | ||
| 333 | localLTags | ||
| 334 | ) | ||
| 335 | |||
| 336 | this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH) | ||
| 337 | }) | ||
| 338 | |||
| 339 | muxingSession.on('duration-exceeded', ({ videoUUID }) => { | ||
| 340 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) | ||
| 341 | |||
| 342 | this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED) | ||
| 343 | }) | ||
| 344 | |||
| 345 | muxingSession.on('quota-exceeded', ({ videoUUID }) => { | ||
| 346 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) | ||
| 347 | |||
| 348 | this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED) | ||
| 349 | }) | ||
| 350 | |||
| 351 | muxingSession.on('transcoding-error', ({ videoUUID }) => { | ||
| 352 | this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR) | ||
| 353 | }) | ||
| 354 | |||
| 355 | muxingSession.on('transcoding-end', ({ videoUUID }) => { | ||
| 356 | this.onMuxingFFmpegEnd(videoUUID, sessionId) | ||
| 357 | }) | ||
| 358 | |||
| 359 | muxingSession.on('after-cleanup', ({ videoUUID }) => { | ||
| 360 | this.muxingSessions.delete(sessionId) | ||
| 361 | |||
| 362 | LiveQuotaStore.Instance.removeLive(user.id, sessionId) | ||
| 363 | |||
| 364 | muxingSession.destroy() | ||
| 365 | |||
| 366 | return this.onAfterMuxingCleanup({ videoUUID, liveSession }) | ||
| 367 | .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) | ||
| 368 | }) | ||
| 369 | |||
| 370 | this.muxingSessions.set(sessionId, muxingSession) | ||
| 371 | |||
| 372 | muxingSession.runMuxing() | ||
| 373 | .catch(err => { | ||
| 374 | logger.error('Cannot run muxing.', { err, ...localLTags }) | ||
| 375 | this.abortSession(sessionId) | ||
| 376 | }) | ||
| 377 | } | ||
| 378 | |||
| 379 | private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { | ||
| 380 | const videoId = live.videoId | ||
| 381 | |||
| 382 | try { | ||
| 383 | const video = await VideoModel.loadFull(videoId) | ||
| 384 | |||
| 385 | logger.info('Will publish and federate live %s.', video.url, localLTags) | ||
| 386 | |||
| 387 | video.state = VideoState.PUBLISHED | ||
| 388 | video.publishedAt = new Date() | ||
| 389 | await video.save() | ||
| 390 | |||
| 391 | live.Video = video | ||
| 392 | |||
| 393 | await wait(getLiveSegmentTime(live.latencyMode) * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) | ||
| 394 | |||
| 395 | try { | ||
| 396 | await federateVideoIfNeeded(video, false) | ||
| 397 | } catch (err) { | ||
| 398 | logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }) | ||
| 399 | } | ||
| 400 | |||
| 401 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | ||
| 402 | |||
| 403 | Hooks.runAction('action:live.video.state.updated', { video }) | ||
| 404 | } catch (err) { | ||
| 405 | logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) | ||
| 406 | } | ||
| 407 | } | ||
| 408 | |||
| 409 | private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) { | ||
| 410 | // Session already cleaned up | ||
| 411 | if (!this.videoSessions.has(videoUUID)) return | ||
| 412 | |||
| 413 | this.videoSessions.delete(videoUUID) | ||
| 414 | |||
| 415 | this.saveEndingSession(videoUUID, null) | ||
| 416 | .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) | ||
| 417 | } | ||
| 418 | |||
| 419 | private async onAfterMuxingCleanup (options: { | ||
| 420 | videoUUID: string | ||
| 421 | liveSession?: MVideoLiveSession | ||
| 422 | cleanupNow?: boolean // Default false | ||
| 423 | }) { | ||
| 424 | const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options | ||
| 425 | |||
| 426 | logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID)) | ||
| 427 | |||
| 428 | try { | ||
| 429 | const fullVideo = await VideoModel.loadFull(videoUUID) | ||
| 430 | if (!fullVideo) return | ||
| 431 | |||
| 432 | const live = await VideoLiveModel.loadByVideoId(fullVideo.id) | ||
| 433 | |||
| 434 | const liveSession = liveSessionArg ?? await VideoLiveSessionModel.findLatestSessionOf(fullVideo.id) | ||
| 435 | |||
| 436 | // On server restart during a live | ||
| 437 | if (!liveSession.endDate) { | ||
| 438 | liveSession.endDate = new Date() | ||
| 439 | await liveSession.save() | ||
| 440 | } | ||
| 441 | |||
| 442 | JobQueue.Instance.createJobAsync({ | ||
| 443 | type: 'video-live-ending', | ||
| 444 | payload: { | ||
| 445 | videoId: fullVideo.id, | ||
| 446 | |||
| 447 | replayDirectory: live.saveReplay | ||
| 448 | ? await this.findReplayDirectory(fullVideo) | ||
| 449 | : undefined, | ||
| 450 | |||
| 451 | liveSessionId: liveSession.id, | ||
| 452 | streamingPlaylistId: fullVideo.getHLSPlaylist()?.id, | ||
| 453 | |||
| 454 | publishedAt: fullVideo.publishedAt.toISOString() | ||
| 455 | }, | ||
| 456 | |||
| 457 | delay: cleanupNow | ||
| 458 | ? 0 | ||
| 459 | : VIDEO_LIVE.CLEANUP_DELAY | ||
| 460 | }) | ||
| 461 | |||
| 462 | fullVideo.state = live.permanentLive | ||
| 463 | ? VideoState.WAITING_FOR_LIVE | ||
| 464 | : VideoState.LIVE_ENDED | ||
| 465 | |||
| 466 | await fullVideo.save() | ||
| 467 | |||
| 468 | PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) | ||
| 469 | |||
| 470 | await federateVideoIfNeeded(fullVideo, false) | ||
| 471 | |||
| 472 | Hooks.runAction('action:live.video.state.updated', { video: fullVideo }) | ||
| 473 | } catch (err) { | ||
| 474 | logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) }) | ||
| 475 | } | ||
| 476 | } | ||
| 477 | |||
| 478 | private async handleBrokenLives () { | ||
| 479 | await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' }) | ||
| 480 | |||
| 481 | const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() | ||
| 482 | |||
| 483 | for (const uuid of videoUUIDs) { | ||
| 484 | await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true }) | ||
| 485 | } | ||
| 486 | } | ||
| 487 | |||
| 488 | private async findReplayDirectory (video: MVideo) { | ||
| 489 | const directory = getLiveReplayBaseDirectory(video) | ||
| 490 | const files = await readdir(directory) | ||
| 491 | |||
| 492 | if (files.length === 0) return undefined | ||
| 493 | |||
| 494 | return join(directory, files.sort().reverse()[0]) | ||
| 495 | } | ||
| 496 | |||
| 497 | private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) { | ||
| 498 | const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
| 499 | |||
| 500 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED | ||
| 501 | ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio }) | ||
| 502 | : [] | ||
| 503 | |||
| 504 | if (resolutionsEnabled.length === 0) { | ||
| 505 | return [ originResolution ] | ||
| 506 | } | ||
| 507 | |||
| 508 | return resolutionsEnabled | ||
| 509 | } | ||
| 510 | |||
| 511 | private async saveStartingSession (videoLive: MVideoLiveVideoWithSetting) { | ||
| 512 | const replaySettings = videoLive.saveReplay | ||
| 513 | ? new VideoLiveReplaySettingModel({ | ||
| 514 | privacy: videoLive.ReplaySetting.privacy | ||
| 515 | }) | ||
| 516 | : null | ||
| 517 | |||
| 518 | return sequelizeTypescript.transaction(async t => { | ||
| 519 | if (videoLive.saveReplay) { | ||
| 520 | await replaySettings.save({ transaction: t }) | ||
| 521 | } | ||
| 522 | |||
| 523 | return VideoLiveSessionModel.create({ | ||
| 524 | startDate: new Date(), | ||
| 525 | liveVideoId: videoLive.videoId, | ||
| 526 | saveReplay: videoLive.saveReplay, | ||
| 527 | replaySettingId: videoLive.saveReplay ? replaySettings.id : null, | ||
| 528 | endingProcessed: false | ||
| 529 | }, { transaction: t }) | ||
| 530 | }) | ||
| 531 | } | ||
| 532 | |||
| 533 | private async saveEndingSession (videoUUID: string, error: LiveVideoError | null) { | ||
| 534 | const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID) | ||
| 535 | if (!liveSession) return | ||
| 536 | |||
| 537 | liveSession.endDate = new Date() | ||
| 538 | liveSession.error = error | ||
| 539 | |||
| 540 | return liveSession.save() | ||
| 541 | } | ||
| 542 | |||
| 543 | static get Instance () { | ||
| 544 | return this.instance || (this.instance = new this()) | ||
| 545 | } | ||
| 546 | } | ||
| 547 | |||
| 548 | // --------------------------------------------------------------------------- | ||
| 549 | |||
| 550 | export { | ||
| 551 | LiveManager | ||
| 552 | } | ||
diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts deleted file mode 100644 index 44539faaa..000000000 --- a/server/lib/live/live-quota-store.ts +++ /dev/null | |||
| @@ -1,48 +0,0 @@ | |||
| 1 | class LiveQuotaStore { | ||
| 2 | |||
| 3 | private static instance: LiveQuotaStore | ||
| 4 | |||
| 5 | private readonly livesPerUser = new Map<number, { sessionId: string, size: number }[]>() | ||
| 6 | |||
| 7 | private constructor () { | ||
| 8 | } | ||
| 9 | |||
| 10 | addNewLive (userId: number, sessionId: string) { | ||
| 11 | if (!this.livesPerUser.has(userId)) { | ||
| 12 | this.livesPerUser.set(userId, []) | ||
| 13 | } | ||
| 14 | |||
| 15 | const currentUserLive = { sessionId, size: 0 } | ||
| 16 | const livesOfUser = this.livesPerUser.get(userId) | ||
| 17 | livesOfUser.push(currentUserLive) | ||
| 18 | } | ||
| 19 | |||
| 20 | removeLive (userId: number, sessionId: string) { | ||
| 21 | const newLivesPerUser = this.livesPerUser.get(userId) | ||
| 22 | .filter(o => o.sessionId !== sessionId) | ||
| 23 | |||
| 24 | this.livesPerUser.set(userId, newLivesPerUser) | ||
| 25 | } | ||
| 26 | |||
| 27 | addQuotaTo (userId: number, sessionId: string, size: number) { | ||
| 28 | const lives = this.livesPerUser.get(userId) | ||
| 29 | const live = lives.find(l => l.sessionId === sessionId) | ||
| 30 | |||
| 31 | live.size += size | ||
| 32 | } | ||
| 33 | |||
| 34 | getLiveQuotaOf (userId: number) { | ||
| 35 | const currentLives = this.livesPerUser.get(userId) | ||
| 36 | if (!currentLives) return 0 | ||
| 37 | |||
| 38 | return currentLives.reduce((sum, obj) => sum + obj.size, 0) | ||
| 39 | } | ||
| 40 | |||
| 41 | static get Instance () { | ||
| 42 | return this.instance || (this.instance = new this()) | ||
| 43 | } | ||
| 44 | } | ||
| 45 | |||
| 46 | export { | ||
| 47 | LiveQuotaStore | ||
| 48 | } | ||
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts deleted file mode 100644 index 8253c0274..000000000 --- a/server/lib/live/live-segment-sha-store.ts +++ /dev/null | |||
| @@ -1,95 +0,0 @@ | |||
| 1 | import { rename, writeJson } from 'fs-extra' | ||
| 2 | import PQueue from 'p-queue' | ||
| 3 | import { basename } from 'path' | ||
| 4 | import { mapToJSON } from '@server/helpers/core-utils' | ||
| 5 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 6 | import { MStreamingPlaylistVideo } from '@server/types/models' | ||
| 7 | import { buildSha256Segment } from '../hls' | ||
| 8 | import { storeHLSFileFromPath } from '../object-storage' | ||
| 9 | |||
| 10 | const lTags = loggerTagsFactory('live') | ||
| 11 | |||
| 12 | class LiveSegmentShaStore { | ||
| 13 | |||
| 14 | private readonly segmentsSha256 = new Map<string, string>() | ||
| 15 | |||
| 16 | private readonly videoUUID: string | ||
| 17 | |||
| 18 | private readonly sha256Path: string | ||
| 19 | private readonly sha256PathTMP: string | ||
| 20 | |||
| 21 | private readonly streamingPlaylist: MStreamingPlaylistVideo | ||
| 22 | private readonly sendToObjectStorage: boolean | ||
| 23 | private readonly writeQueue = new PQueue({ concurrency: 1 }) | ||
| 24 | |||
| 25 | constructor (options: { | ||
| 26 | videoUUID: string | ||
| 27 | sha256Path: string | ||
| 28 | streamingPlaylist: MStreamingPlaylistVideo | ||
| 29 | sendToObjectStorage: boolean | ||
| 30 | }) { | ||
| 31 | this.videoUUID = options.videoUUID | ||
| 32 | |||
| 33 | this.sha256Path = options.sha256Path | ||
| 34 | this.sha256PathTMP = options.sha256Path + '.tmp' | ||
| 35 | |||
| 36 | this.streamingPlaylist = options.streamingPlaylist | ||
| 37 | this.sendToObjectStorage = options.sendToObjectStorage | ||
| 38 | } | ||
| 39 | |||
| 40 | async addSegmentSha (segmentPath: string) { | ||
| 41 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID)) | ||
| 42 | |||
| 43 | const shaResult = await buildSha256Segment(segmentPath) | ||
| 44 | |||
| 45 | const segmentName = basename(segmentPath) | ||
| 46 | this.segmentsSha256.set(segmentName, shaResult) | ||
| 47 | |||
| 48 | try { | ||
| 49 | await this.writeToDisk() | ||
| 50 | } catch (err) { | ||
| 51 | logger.error('Cannot write sha segments to disk.', { err }) | ||
| 52 | } | ||
| 53 | } | ||
| 54 | |||
| 55 | async removeSegmentSha (segmentPath: string) { | ||
| 56 | const segmentName = basename(segmentPath) | ||
| 57 | |||
| 58 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) | ||
| 59 | |||
| 60 | if (!this.segmentsSha256.has(segmentName)) { | ||
| 61 | logger.warn( | ||
| 62 | 'Unknown segment in live segment hash store for video %s and segment %s.', | ||
| 63 | this.videoUUID, segmentPath, lTags(this.videoUUID) | ||
| 64 | ) | ||
| 65 | return | ||
| 66 | } | ||
| 67 | |||
| 68 | this.segmentsSha256.delete(segmentName) | ||
| 69 | |||
| 70 | await this.writeToDisk() | ||
| 71 | } | ||
| 72 | |||
| 73 | private writeToDisk () { | ||
| 74 | return this.writeQueue.add(async () => { | ||
| 75 | logger.debug(`Writing segment sha JSON ${this.sha256Path} of ${this.videoUUID} on disk.`, lTags(this.videoUUID)) | ||
| 76 | |||
| 77 | // Atomic write: use rename instead of move that is not atomic | ||
| 78 | await writeJson(this.sha256PathTMP, mapToJSON(this.segmentsSha256)) | ||
| 79 | await rename(this.sha256PathTMP, this.sha256Path) | ||
| 80 | |||
| 81 | if (this.sendToObjectStorage) { | ||
| 82 | const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) | ||
| 83 | |||
| 84 | if (this.streamingPlaylist.segmentsSha256Url !== url) { | ||
| 85 | this.streamingPlaylist.segmentsSha256Url = url | ||
| 86 | await this.streamingPlaylist.save() | ||
| 87 | } | ||
| 88 | } | ||
| 89 | }) | ||
| 90 | } | ||
| 91 | } | ||
| 92 | |||
| 93 | export { | ||
| 94 | LiveSegmentShaStore | ||
| 95 | } | ||
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts deleted file mode 100644 index 3fb3ce1ce..000000000 --- a/server/lib/live/live-utils.ts +++ /dev/null | |||
| @@ -1,99 +0,0 @@ | |||
| 1 | import { pathExists, readdir, remove } from 'fs-extra' | ||
| 2 | import { basename, join } from 'path' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { VIDEO_LIVE } from '@server/initializers/constants' | ||
| 5 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' | ||
| 6 | import { LiveVideoLatencyMode, VideoStorage } from '@shared/models' | ||
| 7 | import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage' | ||
| 8 | import { getLiveDirectory } from '../paths' | ||
| 9 | |||
| 10 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | ||
| 11 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | ||
| 12 | |||
| 13 | return 'concat-' + num[1] + '.ts' | ||
| 14 | } | ||
| 15 | |||
| 16 | async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | ||
| 17 | await cleanupTMPLiveFiles(video, streamingPlaylist) | ||
| 18 | |||
| 19 | await streamingPlaylist.destroy() | ||
| 20 | } | ||
| 21 | |||
| 22 | async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | ||
| 23 | const hlsDirectory = getLiveDirectory(video) | ||
| 24 | |||
| 25 | // We uploaded files to object storage too, remove them | ||
| 26 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
| 27 | await removeHLSObjectStorage(streamingPlaylist.withVideo(video)) | ||
| 28 | } | ||
| 29 | |||
| 30 | await remove(hlsDirectory) | ||
| 31 | |||
| 32 | await streamingPlaylist.destroy() | ||
| 33 | } | ||
| 34 | |||
| 35 | async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | ||
| 36 | await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video)) | ||
| 37 | |||
| 38 | await cleanupTMPLiveFilesFromFilesystem(video) | ||
| 39 | } | ||
| 40 | |||
| 41 | function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) { | ||
| 42 | if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { | ||
| 43 | return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY | ||
| 44 | } | ||
| 45 | |||
| 46 | return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY | ||
| 47 | } | ||
| 48 | |||
| 49 | export { | ||
| 50 | cleanupAndDestroyPermanentLive, | ||
| 51 | cleanupUnsavedNormalLive, | ||
| 52 | cleanupTMPLiveFiles, | ||
| 53 | getLiveSegmentTime, | ||
| 54 | buildConcatenatedName | ||
| 55 | } | ||
| 56 | |||
| 57 | // --------------------------------------------------------------------------- | ||
| 58 | |||
| 59 | function isTMPLiveFile (name: string) { | ||
| 60 | return name.endsWith('.ts') || | ||
| 61 | name.endsWith('.m3u8') || | ||
| 62 | name.endsWith('.json') || | ||
| 63 | name.endsWith('.mpd') || | ||
| 64 | name.endsWith('.m4s') || | ||
| 65 | name.endsWith('.tmp') | ||
| 66 | } | ||
| 67 | |||
| 68 | async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) { | ||
| 69 | const hlsDirectory = getLiveDirectory(video) | ||
| 70 | |||
| 71 | if (!await pathExists(hlsDirectory)) return | ||
| 72 | |||
| 73 | logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory) | ||
| 74 | |||
| 75 | const files = await readdir(hlsDirectory) | ||
| 76 | |||
| 77 | for (const filename of files) { | ||
| 78 | if (isTMPLiveFile(filename)) { | ||
| 79 | const p = join(hlsDirectory, filename) | ||
| 80 | |||
| 81 | remove(p) | ||
| 82 | .catch(err => logger.error('Cannot remove %s.', p, { err })) | ||
| 83 | } | ||
| 84 | } | ||
| 85 | } | ||
| 86 | |||
| 87 | async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { | ||
| 88 | if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return | ||
| 89 | |||
| 90 | logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid) | ||
| 91 | |||
| 92 | const keys = await listHLSFileKeysOf(streamingPlaylist) | ||
| 93 | |||
| 94 | for (const key of keys) { | ||
| 95 | if (isTMPLiveFile(key)) { | ||
| 96 | await removeHLSFileObjectStorageByFullKey(key) | ||
| 97 | } | ||
| 98 | } | ||
| 99 | } | ||
diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts deleted file mode 100644 index c4d1b59ec..000000000 --- a/server/lib/live/shared/index.ts +++ /dev/null | |||
| @@ -1 +0,0 @@ | |||
| 1 | export * from './muxing-session' | ||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts deleted file mode 100644 index 02691b651..000000000 --- a/server/lib/live/shared/muxing-session.ts +++ /dev/null | |||
| @@ -1,518 +0,0 @@ | |||
| 1 | import { mapSeries } from 'bluebird' | ||
| 2 | import { FSWatcher, watch } from 'chokidar' | ||
| 3 | import { EventEmitter } from 'events' | ||
| 4 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | ||
| 5 | import PQueue from 'p-queue' | ||
| 6 | import { basename, join } from 'path' | ||
| 7 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
| 8 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | ||
| 9 | import { CONFIG } from '@server/initializers/config' | ||
| 10 | import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants' | ||
| 11 | import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFileFromPath } from '@server/lib/object-storage' | ||
| 12 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 13 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | ||
| 14 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | ||
| 15 | import { VideoStorage, VideoStreamingPlaylistType } from '@shared/models' | ||
| 16 | import { | ||
| 17 | generateHLSMasterPlaylistFilename, | ||
| 18 | generateHlsSha256SegmentsFilename, | ||
| 19 | getLiveDirectory, | ||
| 20 | getLiveReplayBaseDirectory | ||
| 21 | } from '../../paths' | ||
| 22 | import { isAbleToUploadVideo } from '../../user' | ||
| 23 | import { LiveQuotaStore } from '../live-quota-store' | ||
| 24 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | ||
| 25 | import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils' | ||
| 26 | import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper' | ||
| 27 | |||
| 28 | import memoizee = require('memoizee') | ||
| 29 | interface MuxingSessionEvents { | ||
| 30 | 'live-ready': (options: { videoUUID: string }) => void | ||
| 31 | |||
| 32 | 'bad-socket-health': (options: { videoUUID: string }) => void | ||
| 33 | 'duration-exceeded': (options: { videoUUID: string }) => void | ||
| 34 | 'quota-exceeded': (options: { videoUUID: string }) => void | ||
| 35 | |||
| 36 | 'transcoding-end': (options: { videoUUID: string }) => void | ||
| 37 | 'transcoding-error': (options: { videoUUID: string }) => void | ||
| 38 | |||
| 39 | 'after-cleanup': (options: { videoUUID: string }) => void | ||
| 40 | } | ||
| 41 | |||
| 42 | declare interface MuxingSession { | ||
| 43 | on<U extends keyof MuxingSessionEvents>( | ||
| 44 | event: U, listener: MuxingSessionEvents[U] | ||
| 45 | ): this | ||
| 46 | |||
| 47 | emit<U extends keyof MuxingSessionEvents>( | ||
| 48 | event: U, ...args: Parameters<MuxingSessionEvents[U]> | ||
| 49 | ): boolean | ||
| 50 | } | ||
| 51 | |||
| 52 | class MuxingSession extends EventEmitter { | ||
| 53 | |||
| 54 | private transcodingWrapper: AbstractTranscodingWrapper | ||
| 55 | |||
| 56 | private readonly context: any | ||
| 57 | private readonly user: MUserId | ||
| 58 | private readonly sessionId: string | ||
| 59 | private readonly videoLive: MVideoLiveVideo | ||
| 60 | |||
| 61 | private readonly inputLocalUrl: string | ||
| 62 | private readonly inputPublicUrl: string | ||
| 63 | |||
| 64 | private readonly fps: number | ||
| 65 | private readonly allResolutions: number[] | ||
| 66 | |||
| 67 | private readonly bitrate: number | ||
| 68 | private readonly ratio: number | ||
| 69 | |||
| 70 | private readonly hasAudio: boolean | ||
| 71 | |||
| 72 | private readonly videoUUID: string | ||
| 73 | private readonly saveReplay: boolean | ||
| 74 | |||
| 75 | private readonly outDirectory: string | ||
| 76 | private readonly replayDirectory: string | ||
| 77 | |||
| 78 | private readonly lTags: LoggerTagsFn | ||
| 79 | |||
| 80 | // Path -> Queue | ||
| 81 | private readonly objectStorageSendQueues = new Map<string, PQueue>() | ||
| 82 | |||
| 83 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | ||
| 84 | |||
| 85 | private streamingPlaylist: MStreamingPlaylistVideo | ||
| 86 | private liveSegmentShaStore: LiveSegmentShaStore | ||
| 87 | |||
| 88 | private filesWatcher: FSWatcher | ||
| 89 | |||
| 90 | private masterPlaylistCreated = false | ||
| 91 | private liveReady = false | ||
| 92 | |||
| 93 | private aborted = false | ||
| 94 | |||
| 95 | private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { | ||
| 96 | return isAbleToUploadVideo(userId, 1000) | ||
| 97 | }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) | ||
| 98 | |||
| 99 | private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { | ||
| 100 | return this.hasClientSocketInBadHealth(sessionId) | ||
| 101 | }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) | ||
| 102 | |||
| 103 | constructor (options: { | ||
| 104 | context: any | ||
| 105 | user: MUserId | ||
| 106 | sessionId: string | ||
| 107 | videoLive: MVideoLiveVideo | ||
| 108 | |||
| 109 | inputLocalUrl: string | ||
| 110 | inputPublicUrl: string | ||
| 111 | |||
| 112 | fps: number | ||
| 113 | bitrate: number | ||
| 114 | ratio: number | ||
| 115 | allResolutions: number[] | ||
| 116 | hasAudio: boolean | ||
| 117 | }) { | ||
| 118 | super() | ||
| 119 | |||
| 120 | this.context = options.context | ||
| 121 | this.user = options.user | ||
| 122 | this.sessionId = options.sessionId | ||
| 123 | this.videoLive = options.videoLive | ||
| 124 | |||
| 125 | this.inputLocalUrl = options.inputLocalUrl | ||
| 126 | this.inputPublicUrl = options.inputPublicUrl | ||
| 127 | |||
| 128 | this.fps = options.fps | ||
| 129 | |||
| 130 | this.bitrate = options.bitrate | ||
| 131 | this.ratio = options.ratio | ||
| 132 | |||
| 133 | this.hasAudio = options.hasAudio | ||
| 134 | |||
| 135 | this.allResolutions = options.allResolutions | ||
| 136 | |||
| 137 | this.videoUUID = this.videoLive.Video.uuid | ||
| 138 | |||
| 139 | this.saveReplay = this.videoLive.saveReplay | ||
| 140 | |||
| 141 | this.outDirectory = getLiveDirectory(this.videoLive.Video) | ||
| 142 | this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) | ||
| 143 | |||
| 144 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) | ||
| 145 | } | ||
| 146 | |||
| 147 | async runMuxing () { | ||
| 148 | this.streamingPlaylist = await this.createLivePlaylist() | ||
| 149 | |||
| 150 | this.createLiveShaStore() | ||
| 151 | this.createFiles() | ||
| 152 | |||
| 153 | await this.prepareDirectories() | ||
| 154 | |||
| 155 | this.transcodingWrapper = this.buildTranscodingWrapper() | ||
| 156 | |||
| 157 | this.transcodingWrapper.on('end', () => this.onTranscodedEnded()) | ||
| 158 | this.transcodingWrapper.on('error', () => this.onTranscodingError()) | ||
| 159 | |||
| 160 | await this.transcodingWrapper.run() | ||
| 161 | |||
| 162 | this.filesWatcher = watch(this.outDirectory, { depth: 0 }) | ||
| 163 | |||
| 164 | this.watchMasterFile() | ||
| 165 | this.watchTSFiles() | ||
| 166 | } | ||
| 167 | |||
| 168 | abort () { | ||
| 169 | if (!this.transcodingWrapper) return | ||
| 170 | |||
| 171 | this.aborted = true | ||
| 172 | this.transcodingWrapper.abort() | ||
| 173 | } | ||
| 174 | |||
| 175 | destroy () { | ||
| 176 | this.removeAllListeners() | ||
| 177 | this.isAbleToUploadVideoWithCache.clear() | ||
| 178 | this.hasClientSocketInBadHealthWithCache.clear() | ||
| 179 | } | ||
| 180 | |||
| 181 | private watchMasterFile () { | ||
| 182 | this.filesWatcher.on('add', async path => { | ||
| 183 | if (path !== join(this.outDirectory, this.streamingPlaylist.playlistFilename)) return | ||
| 184 | if (this.masterPlaylistCreated === true) return | ||
| 185 | |||
| 186 | try { | ||
| 187 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
| 188 | const masterContent = await readFile(path, 'utf-8') | ||
| 189 | logger.debug('Uploading live master playlist on object storage for %s', this.videoUUID, { masterContent, ...this.lTags() }) | ||
| 190 | |||
| 191 | const url = await storeHLSFileFromContent(this.streamingPlaylist, this.streamingPlaylist.playlistFilename, masterContent) | ||
| 192 | |||
| 193 | this.streamingPlaylist.playlistUrl = url | ||
| 194 | } | ||
| 195 | |||
| 196 | this.streamingPlaylist.assignP2PMediaLoaderInfoHashes(this.videoLive.Video, this.allResolutions) | ||
| 197 | |||
| 198 | await this.streamingPlaylist.save() | ||
| 199 | } catch (err) { | ||
| 200 | logger.error('Cannot update streaming playlist.', { err, ...this.lTags() }) | ||
| 201 | } | ||
| 202 | |||
| 203 | this.masterPlaylistCreated = true | ||
| 204 | |||
| 205 | logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags()) | ||
| 206 | }) | ||
| 207 | } | ||
| 208 | |||
| 209 | private watchTSFiles () { | ||
| 210 | const startStreamDateTime = new Date().getTime() | ||
| 211 | |||
| 212 | const addHandler = async (segmentPath: string) => { | ||
| 213 | if (segmentPath.endsWith('.ts') !== true) return | ||
| 214 | |||
| 215 | logger.debug('Live add handler of TS file %s.', segmentPath, this.lTags()) | ||
| 216 | |||
| 217 | const playlistId = this.getPlaylistIdFromTS(segmentPath) | ||
| 218 | |||
| 219 | const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] | ||
| 220 | this.processSegments(segmentsToProcess) | ||
| 221 | |||
| 222 | this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] | ||
| 223 | |||
| 224 | if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { | ||
| 225 | this.emit('bad-socket-health', { videoUUID: this.videoUUID }) | ||
| 226 | return | ||
| 227 | } | ||
| 228 | |||
| 229 | // Duration constraint check | ||
| 230 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | ||
| 231 | this.emit('duration-exceeded', { videoUUID: this.videoUUID }) | ||
| 232 | return | ||
| 233 | } | ||
| 234 | |||
| 235 | // Check user quota if the user enabled replay saving | ||
| 236 | if (await this.isQuotaExceeded(segmentPath) === true) { | ||
| 237 | this.emit('quota-exceeded', { videoUUID: this.videoUUID }) | ||
| 238 | } | ||
| 239 | } | ||
| 240 | |||
| 241 | const deleteHandler = async (segmentPath: string) => { | ||
| 242 | if (segmentPath.endsWith('.ts') !== true) return | ||
| 243 | |||
| 244 | logger.debug('Live delete handler of TS file %s.', segmentPath, this.lTags()) | ||
| 245 | |||
| 246 | try { | ||
| 247 | await this.liveSegmentShaStore.removeSegmentSha(segmentPath) | ||
| 248 | } catch (err) { | ||
| 249 | logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) | ||
| 250 | } | ||
| 251 | |||
| 252 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
| 253 | try { | ||
| 254 | await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath) | ||
| 255 | } catch (err) { | ||
| 256 | logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) | ||
| 257 | } | ||
| 258 | } | ||
| 259 | } | ||
| 260 | |||
| 261 | this.filesWatcher.on('add', p => addHandler(p)) | ||
| 262 | this.filesWatcher.on('unlink', p => deleteHandler(p)) | ||
| 263 | } | ||
| 264 | |||
| 265 | private async isQuotaExceeded (segmentPath: string) { | ||
| 266 | if (this.saveReplay !== true) return false | ||
| 267 | if (this.aborted) return false | ||
| 268 | |||
| 269 | try { | ||
| 270 | const segmentStat = await stat(segmentPath) | ||
| 271 | |||
| 272 | LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.sessionId, segmentStat.size) | ||
| 273 | |||
| 274 | const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) | ||
| 275 | |||
| 276 | return canUpload !== true | ||
| 277 | } catch (err) { | ||
| 278 | logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags() }) | ||
| 279 | } | ||
| 280 | } | ||
| 281 | |||
| 282 | private createFiles () { | ||
| 283 | for (let i = 0; i < this.allResolutions.length; i++) { | ||
| 284 | const resolution = this.allResolutions[i] | ||
| 285 | |||
| 286 | const file = new VideoFileModel({ | ||
| 287 | resolution, | ||
| 288 | size: -1, | ||
| 289 | extname: '.ts', | ||
| 290 | infoHash: null, | ||
| 291 | fps: this.fps, | ||
| 292 | storage: this.streamingPlaylist.storage, | ||
| 293 | videoStreamingPlaylistId: this.streamingPlaylist.id | ||
| 294 | }) | ||
| 295 | |||
| 296 | VideoFileModel.customUpsert(file, 'streaming-playlist', null) | ||
| 297 | .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags() })) | ||
| 298 | } | ||
| 299 | } | ||
| 300 | |||
| 301 | private async prepareDirectories () { | ||
| 302 | await ensureDir(this.outDirectory) | ||
| 303 | |||
| 304 | if (this.videoLive.saveReplay === true) { | ||
| 305 | await ensureDir(this.replayDirectory) | ||
| 306 | } | ||
| 307 | } | ||
| 308 | |||
| 309 | private isDurationConstraintValid (streamingStartTime: number) { | ||
| 310 | const maxDuration = CONFIG.LIVE.MAX_DURATION | ||
| 311 | // No limit | ||
| 312 | if (maxDuration < 0) return true | ||
| 313 | |||
| 314 | const now = new Date().getTime() | ||
| 315 | const max = streamingStartTime + maxDuration | ||
| 316 | |||
| 317 | return now <= max | ||
| 318 | } | ||
| 319 | |||
| 320 | private processSegments (segmentPaths: string[]) { | ||
| 321 | mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment)) | ||
| 322 | .catch(err => { | ||
| 323 | if (this.aborted) return | ||
| 324 | |||
| 325 | logger.error('Cannot process segments', { err, ...this.lTags() }) | ||
| 326 | }) | ||
| 327 | } | ||
| 328 | |||
| 329 | private async processSegment (segmentPath: string) { | ||
| 330 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | ||
| 331 | await this.liveSegmentShaStore.addSegmentSha(segmentPath) | ||
| 332 | |||
| 333 | if (this.saveReplay) { | ||
| 334 | await this.addSegmentToReplay(segmentPath) | ||
| 335 | } | ||
| 336 | |||
| 337 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
| 338 | try { | ||
| 339 | await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) | ||
| 340 | |||
| 341 | await this.processM3U8ToObjectStorage(segmentPath) | ||
| 342 | } catch (err) { | ||
| 343 | logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() }) | ||
| 344 | } | ||
| 345 | } | ||
| 346 | |||
| 347 | // Master playlist and segment JSON file are created, live is ready | ||
| 348 | if (this.masterPlaylistCreated && !this.liveReady) { | ||
| 349 | this.liveReady = true | ||
| 350 | |||
| 351 | this.emit('live-ready', { videoUUID: this.videoUUID }) | ||
| 352 | } | ||
| 353 | } | ||
| 354 | |||
| 355 | private async processM3U8ToObjectStorage (segmentPath: string) { | ||
| 356 | const m3u8Path = join(this.outDirectory, this.getPlaylistNameFromTS(segmentPath)) | ||
| 357 | |||
| 358 | logger.debug('Process M3U8 file %s.', m3u8Path, this.lTags()) | ||
| 359 | |||
| 360 | const segmentName = basename(segmentPath) | ||
| 361 | |||
| 362 | const playlistContent = await readFile(m3u8Path, 'utf-8') | ||
| 363 | // Remove new chunk references, that will be processed later | ||
| 364 | const filteredPlaylistContent = playlistContent.substring(0, playlistContent.lastIndexOf(segmentName) + segmentName.length) + '\n' | ||
| 365 | |||
| 366 | try { | ||
| 367 | if (!this.objectStorageSendQueues.has(m3u8Path)) { | ||
| 368 | this.objectStorageSendQueues.set(m3u8Path, new PQueue({ concurrency: 1 })) | ||
| 369 | } | ||
| 370 | |||
| 371 | const queue = this.objectStorageSendQueues.get(m3u8Path) | ||
| 372 | await queue.add(() => storeHLSFileFromContent(this.streamingPlaylist, m3u8Path, filteredPlaylistContent)) | ||
| 373 | } catch (err) { | ||
| 374 | logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() }) | ||
| 375 | } | ||
| 376 | } | ||
| 377 | |||
| 378 | private onTranscodingError () { | ||
| 379 | this.emit('transcoding-error', ({ videoUUID: this.videoUUID })) | ||
| 380 | } | ||
| 381 | |||
| 382 | private onTranscodedEnded () { | ||
| 383 | this.emit('transcoding-end', ({ videoUUID: this.videoUUID })) | ||
| 384 | |||
| 385 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputLocalUrl, this.lTags()) | ||
| 386 | |||
| 387 | setTimeout(() => { | ||
| 388 | // Wait latest segments generation, and close watchers | ||
| 389 | |||
| 390 | const promise = this.filesWatcher?.close() || Promise.resolve() | ||
| 391 | promise | ||
| 392 | .then(() => { | ||
| 393 | // Process remaining segments hash | ||
| 394 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | ||
| 395 | this.processSegments(this.segmentsToProcessPerPlaylist[key]) | ||
| 396 | } | ||
| 397 | }) | ||
| 398 | .catch(err => { | ||
| 399 | logger.error( | ||
| 400 | 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, | ||
| 401 | { err, ...this.lTags() } | ||
| 402 | ) | ||
| 403 | }) | ||
| 404 | |||
| 405 | this.emit('after-cleanup', { videoUUID: this.videoUUID }) | ||
| 406 | }, 1000) | ||
| 407 | } | ||
| 408 | |||
| 409 | private hasClientSocketInBadHealth (sessionId: string) { | ||
| 410 | const rtmpSession = this.context.sessions.get(sessionId) | ||
| 411 | |||
| 412 | if (!rtmpSession) { | ||
| 413 | logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags()) | ||
| 414 | return | ||
| 415 | } | ||
| 416 | |||
| 417 | for (const playerSessionId of rtmpSession.players) { | ||
| 418 | const playerSession = this.context.sessions.get(playerSessionId) | ||
| 419 | |||
| 420 | if (!playerSession) { | ||
| 421 | logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags()) | ||
| 422 | continue | ||
| 423 | } | ||
| 424 | |||
| 425 | if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { | ||
| 426 | return true | ||
| 427 | } | ||
| 428 | } | ||
| 429 | |||
| 430 | return false | ||
| 431 | } | ||
| 432 | |||
| 433 | private async addSegmentToReplay (segmentPath: string) { | ||
| 434 | const segmentName = basename(segmentPath) | ||
| 435 | const dest = join(this.replayDirectory, buildConcatenatedName(segmentName)) | ||
| 436 | |||
| 437 | try { | ||
| 438 | const data = await readFile(segmentPath) | ||
| 439 | |||
| 440 | await appendFile(dest, data) | ||
| 441 | } catch (err) { | ||
| 442 | logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags() }) | ||
| 443 | } | ||
| 444 | } | ||
| 445 | |||
| 446 | private async createLivePlaylist (): Promise<MStreamingPlaylistVideo> { | ||
| 447 | const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(this.videoLive.Video) | ||
| 448 | |||
| 449 | playlist.playlistFilename = generateHLSMasterPlaylistFilename(true) | ||
| 450 | playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true) | ||
| 451 | |||
| 452 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION | ||
| 453 | playlist.type = VideoStreamingPlaylistType.HLS | ||
| 454 | |||
| 455 | playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED | ||
| 456 | ? VideoStorage.OBJECT_STORAGE | ||
| 457 | : VideoStorage.FILE_SYSTEM | ||
| 458 | |||
| 459 | return playlist.save() | ||
| 460 | } | ||
| 461 | |||
| 462 | private createLiveShaStore () { | ||
| 463 | this.liveSegmentShaStore = new LiveSegmentShaStore({ | ||
| 464 | videoUUID: this.videoLive.Video.uuid, | ||
| 465 | sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename), | ||
| 466 | streamingPlaylist: this.streamingPlaylist, | ||
| 467 | sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED | ||
| 468 | }) | ||
| 469 | } | ||
| 470 | |||
| 471 | private buildTranscodingWrapper () { | ||
| 472 | const options = { | ||
| 473 | streamingPlaylist: this.streamingPlaylist, | ||
| 474 | videoLive: this.videoLive, | ||
| 475 | |||
| 476 | lTags: this.lTags, | ||
| 477 | |||
| 478 | sessionId: this.sessionId, | ||
| 479 | inputLocalUrl: this.inputLocalUrl, | ||
| 480 | inputPublicUrl: this.inputPublicUrl, | ||
| 481 | |||
| 482 | toTranscode: this.allResolutions.map(resolution => ({ | ||
| 483 | resolution, | ||
| 484 | fps: computeOutputFPS({ inputFPS: this.fps, resolution }) | ||
| 485 | })), | ||
| 486 | |||
| 487 | fps: this.fps, | ||
| 488 | bitrate: this.bitrate, | ||
| 489 | ratio: this.ratio, | ||
| 490 | hasAudio: this.hasAudio, | ||
| 491 | |||
| 492 | segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, | ||
| 493 | segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode), | ||
| 494 | |||
| 495 | outDirectory: this.outDirectory | ||
| 496 | } | ||
| 497 | |||
| 498 | return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
| 499 | ? new RemoteTranscodingWrapper(options) | ||
| 500 | : new FFmpegTranscodingWrapper(options) | ||
| 501 | } | ||
| 502 | |||
| 503 | private getPlaylistIdFromTS (segmentPath: string) { | ||
| 504 | const playlistIdMatcher = /^([\d+])-/ | ||
| 505 | |||
| 506 | return basename(segmentPath).match(playlistIdMatcher)[1] | ||
| 507 | } | ||
| 508 | |||
| 509 | private getPlaylistNameFromTS (segmentPath: string) { | ||
| 510 | return `${this.getPlaylistIdFromTS(segmentPath)}.m3u8` | ||
| 511 | } | ||
| 512 | } | ||
| 513 | |||
| 514 | // --------------------------------------------------------------------------- | ||
| 515 | |||
| 516 | export { | ||
| 517 | MuxingSession | ||
| 518 | } | ||
diff --git a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts deleted file mode 100644 index 95168745d..000000000 --- a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts +++ /dev/null | |||
| @@ -1,110 +0,0 @@ | |||
| 1 | import EventEmitter from 'events' | ||
| 2 | import { LoggerTagsFn } from '@server/helpers/logger' | ||
| 3 | import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models' | ||
| 4 | import { LiveVideoError } from '@shared/models' | ||
| 5 | |||
| 6 | interface TranscodingWrapperEvents { | ||
| 7 | 'end': () => void | ||
| 8 | |||
| 9 | 'error': (options: { err: Error }) => void | ||
| 10 | } | ||
| 11 | |||
| 12 | declare interface AbstractTranscodingWrapper { | ||
| 13 | on<U extends keyof TranscodingWrapperEvents>( | ||
| 14 | event: U, listener: TranscodingWrapperEvents[U] | ||
| 15 | ): this | ||
| 16 | |||
| 17 | emit<U extends keyof TranscodingWrapperEvents>( | ||
| 18 | event: U, ...args: Parameters<TranscodingWrapperEvents[U]> | ||
| 19 | ): boolean | ||
| 20 | } | ||
| 21 | |||
| 22 | interface AbstractTranscodingWrapperOptions { | ||
| 23 | streamingPlaylist: MStreamingPlaylistVideo | ||
| 24 | videoLive: MVideoLiveVideo | ||
| 25 | |||
| 26 | lTags: LoggerTagsFn | ||
| 27 | |||
| 28 | sessionId: string | ||
| 29 | inputLocalUrl: string | ||
| 30 | inputPublicUrl: string | ||
| 31 | |||
| 32 | fps: number | ||
| 33 | toTranscode: { | ||
| 34 | resolution: number | ||
| 35 | fps: number | ||
| 36 | }[] | ||
| 37 | |||
| 38 | bitrate: number | ||
| 39 | ratio: number | ||
| 40 | hasAudio: boolean | ||
| 41 | |||
| 42 | segmentListSize: number | ||
| 43 | segmentDuration: number | ||
| 44 | |||
| 45 | outDirectory: string | ||
| 46 | } | ||
| 47 | |||
| 48 | abstract class AbstractTranscodingWrapper extends EventEmitter { | ||
| 49 | protected readonly videoLive: MVideoLiveVideo | ||
| 50 | |||
| 51 | protected readonly toTranscode: { | ||
| 52 | resolution: number | ||
| 53 | fps: number | ||
| 54 | }[] | ||
| 55 | |||
| 56 | protected readonly sessionId: string | ||
| 57 | protected readonly inputLocalUrl: string | ||
| 58 | protected readonly inputPublicUrl: string | ||
| 59 | |||
| 60 | protected readonly fps: number | ||
| 61 | protected readonly bitrate: number | ||
| 62 | protected readonly ratio: number | ||
| 63 | protected readonly hasAudio: boolean | ||
| 64 | |||
| 65 | protected readonly segmentListSize: number | ||
| 66 | protected readonly segmentDuration: number | ||
| 67 | |||
| 68 | protected readonly videoUUID: string | ||
| 69 | |||
| 70 | protected readonly outDirectory: string | ||
| 71 | |||
| 72 | protected readonly lTags: LoggerTagsFn | ||
| 73 | |||
| 74 | protected readonly streamingPlaylist: MStreamingPlaylistVideo | ||
| 75 | |||
| 76 | constructor (options: AbstractTranscodingWrapperOptions) { | ||
| 77 | super() | ||
| 78 | |||
| 79 | this.lTags = options.lTags | ||
| 80 | |||
| 81 | this.videoLive = options.videoLive | ||
| 82 | this.videoUUID = options.videoLive.Video.uuid | ||
| 83 | this.streamingPlaylist = options.streamingPlaylist | ||
| 84 | |||
| 85 | this.sessionId = options.sessionId | ||
| 86 | this.inputLocalUrl = options.inputLocalUrl | ||
| 87 | this.inputPublicUrl = options.inputPublicUrl | ||
| 88 | |||
| 89 | this.fps = options.fps | ||
| 90 | this.toTranscode = options.toTranscode | ||
| 91 | |||
| 92 | this.bitrate = options.bitrate | ||
| 93 | this.ratio = options.ratio | ||
| 94 | this.hasAudio = options.hasAudio | ||
| 95 | |||
| 96 | this.segmentListSize = options.segmentListSize | ||
| 97 | this.segmentDuration = options.segmentDuration | ||
| 98 | |||
| 99 | this.outDirectory = options.outDirectory | ||
| 100 | } | ||
| 101 | |||
| 102 | abstract run (): Promise<void> | ||
| 103 | |||
| 104 | abstract abort (error?: LiveVideoError): void | ||
| 105 | } | ||
| 106 | |||
| 107 | export { | ||
| 108 | AbstractTranscodingWrapper, | ||
| 109 | AbstractTranscodingWrapperOptions | ||
| 110 | } | ||
diff --git a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts deleted file mode 100644 index c6ee8ebf1..000000000 --- a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts +++ /dev/null | |||
| @@ -1,107 +0,0 @@ | |||
| 1 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
| 2 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { CONFIG } from '@server/initializers/config' | ||
| 5 | import { VIDEO_LIVE } from '@server/initializers/constants' | ||
| 6 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' | ||
| 7 | import { FFmpegLive } from '@shared/ffmpeg' | ||
| 8 | import { getLiveSegmentTime } from '../../live-utils' | ||
| 9 | import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' | ||
| 10 | |||
| 11 | export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper { | ||
| 12 | private ffmpegCommand: FfmpegCommand | ||
| 13 | |||
| 14 | private aborted = false | ||
| 15 | private errored = false | ||
| 16 | private ended = false | ||
| 17 | |||
| 18 | async run () { | ||
| 19 | this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED | ||
| 20 | ? await this.buildFFmpegLive().getLiveTranscodingCommand({ | ||
| 21 | inputUrl: this.inputLocalUrl, | ||
| 22 | |||
| 23 | outPath: this.outDirectory, | ||
| 24 | masterPlaylistName: this.streamingPlaylist.playlistFilename, | ||
| 25 | |||
| 26 | segmentListSize: this.segmentListSize, | ||
| 27 | segmentDuration: this.segmentDuration, | ||
| 28 | |||
| 29 | toTranscode: this.toTranscode, | ||
| 30 | |||
| 31 | bitrate: this.bitrate, | ||
| 32 | ratio: this.ratio, | ||
| 33 | |||
| 34 | hasAudio: this.hasAudio | ||
| 35 | }) | ||
| 36 | : this.buildFFmpegLive().getLiveMuxingCommand({ | ||
| 37 | inputUrl: this.inputLocalUrl, | ||
| 38 | outPath: this.outDirectory, | ||
| 39 | |||
| 40 | masterPlaylistName: this.streamingPlaylist.playlistFilename, | ||
| 41 | |||
| 42 | segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, | ||
| 43 | segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode) | ||
| 44 | }) | ||
| 45 | |||
| 46 | logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags()) | ||
| 47 | |||
| 48 | let ffmpegShellCommand: string | ||
| 49 | this.ffmpegCommand.on('start', cmdline => { | ||
| 50 | ffmpegShellCommand = cmdline | ||
| 51 | |||
| 52 | logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) | ||
| 53 | }) | ||
| 54 | |||
| 55 | this.ffmpegCommand.on('error', (err, stdout, stderr) => { | ||
| 56 | this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) | ||
| 57 | }) | ||
| 58 | |||
| 59 | this.ffmpegCommand.on('end', () => { | ||
| 60 | this.onFFmpegEnded() | ||
| 61 | }) | ||
| 62 | |||
| 63 | this.ffmpegCommand.run() | ||
| 64 | } | ||
| 65 | |||
| 66 | abort () { | ||
| 67 | if (this.ended || this.errored || this.aborted) return | ||
| 68 | |||
| 69 | logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags()) | ||
| 70 | |||
| 71 | this.ffmpegCommand.kill('SIGINT') | ||
| 72 | |||
| 73 | this.aborted = true | ||
| 74 | this.emit('end') | ||
| 75 | } | ||
| 76 | |||
| 77 | private onFFmpegError (options: { | ||
| 78 | err: any | ||
| 79 | stdout: string | ||
| 80 | stderr: string | ||
| 81 | ffmpegShellCommand: string | ||
| 82 | }) { | ||
| 83 | const { err, stdout, stderr, ffmpegShellCommand } = options | ||
| 84 | |||
| 85 | // Don't care that we killed the ffmpeg process | ||
| 86 | if (err?.message?.includes('Exiting normally')) return | ||
| 87 | if (this.ended || this.errored || this.aborted) return | ||
| 88 | |||
| 89 | logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) | ||
| 90 | |||
| 91 | this.errored = true | ||
| 92 | this.emit('error', { err }) | ||
| 93 | } | ||
| 94 | |||
| 95 | private onFFmpegEnded () { | ||
| 96 | if (this.ended || this.errored || this.aborted) return | ||
| 97 | |||
| 98 | logger.debug('Live ffmpeg transcoding ended for ' + this.videoUUID, this.lTags()) | ||
| 99 | |||
| 100 | this.ended = true | ||
| 101 | this.emit('end') | ||
| 102 | } | ||
| 103 | |||
| 104 | private buildFFmpegLive () { | ||
| 105 | return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) | ||
| 106 | } | ||
| 107 | } | ||
diff --git a/server/lib/live/shared/transcoding-wrapper/index.ts b/server/lib/live/shared/transcoding-wrapper/index.ts deleted file mode 100644 index ae28fa1ca..000000000 --- a/server/lib/live/shared/transcoding-wrapper/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './abstract-transcoding-wrapper' | ||
| 2 | export * from './ffmpeg-transcoding-wrapper' | ||
| 3 | export * from './remote-transcoding-wrapper' | ||
diff --git a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts deleted file mode 100644 index 2aeeb31fb..000000000 --- a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts +++ /dev/null | |||
| @@ -1,21 +0,0 @@ | |||
| 1 | import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners' | ||
| 2 | import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' | ||
| 3 | |||
| 4 | export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper { | ||
| 5 | async run () { | ||
| 6 | await new LiveRTMPHLSTranscodingJobHandler().create({ | ||
| 7 | rtmpUrl: this.inputPublicUrl, | ||
| 8 | sessionId: this.sessionId, | ||
| 9 | toTranscode: this.toTranscode, | ||
| 10 | video: this.videoLive.Video, | ||
| 11 | outputDirectory: this.outDirectory, | ||
| 12 | playlist: this.streamingPlaylist, | ||
| 13 | segmentListSize: this.segmentListSize, | ||
| 14 | segmentDuration: this.segmentDuration | ||
| 15 | }) | ||
| 16 | } | ||
| 17 | |||
| 18 | abort () { | ||
| 19 | this.emit('end') | ||
| 20 | } | ||
| 21 | } | ||
diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts deleted file mode 100644 index 611e6d0af..000000000 --- a/server/lib/local-actor.ts +++ /dev/null | |||
| @@ -1,102 +0,0 @@ | |||
| 1 | import { remove } from 'fs-extra' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { Transaction } from 'sequelize/types' | ||
| 4 | import { ActorModel } from '@server/models/actor/actor' | ||
| 5 | import { getLowercaseExtension } from '@shared/core-utils' | ||
| 6 | import { buildUUID } from '@shared/extra-utils' | ||
| 7 | import { ActivityPubActorType, ActorImageType } from '@shared/models' | ||
| 8 | import { retryTransactionWrapper } from '../helpers/database-utils' | ||
| 9 | import { CONFIG } from '../initializers/config' | ||
| 10 | import { ACTOR_IMAGES_SIZE, WEBSERVER } from '../initializers/constants' | ||
| 11 | import { sequelizeTypescript } from '../initializers/database' | ||
| 12 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' | ||
| 13 | import { deleteActorImages, updateActorImages } from './activitypub/actors' | ||
| 14 | import { sendUpdateActor } from './activitypub/send' | ||
| 15 | import { processImageFromWorker } from './worker/parent-process' | ||
| 16 | |||
| 17 | export function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { | ||
| 18 | return new ActorModel({ | ||
| 19 | type, | ||
| 20 | url, | ||
| 21 | preferredUsername, | ||
| 22 | publicKey: null, | ||
| 23 | privateKey: null, | ||
| 24 | followersCount: 0, | ||
| 25 | followingCount: 0, | ||
| 26 | inboxUrl: url + '/inbox', | ||
| 27 | outboxUrl: url + '/outbox', | ||
| 28 | sharedInboxUrl: WEBSERVER.URL + '/inbox', | ||
| 29 | followersUrl: url + '/followers', | ||
| 30 | followingUrl: url + '/following' | ||
| 31 | }) as MActor | ||
| 32 | } | ||
| 33 | |||
| 34 | export async function updateLocalActorImageFiles ( | ||
| 35 | accountOrChannel: MAccountDefault | MChannelDefault, | ||
| 36 | imagePhysicalFile: Express.Multer.File, | ||
| 37 | type: ActorImageType | ||
| 38 | ) { | ||
| 39 | const processImageSize = async (imageSize: { width: number, height: number }) => { | ||
| 40 | const extension = getLowercaseExtension(imagePhysicalFile.filename) | ||
| 41 | |||
| 42 | const imageName = buildUUID() + extension | ||
| 43 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName) | ||
| 44 | await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) | ||
| 45 | |||
| 46 | return { | ||
| 47 | imageName, | ||
| 48 | imageSize | ||
| 49 | } | ||
| 50 | } | ||
| 51 | |||
| 52 | const processedImages = await Promise.all(ACTOR_IMAGES_SIZE[type].map(processImageSize)) | ||
| 53 | await remove(imagePhysicalFile.path) | ||
| 54 | |||
| 55 | return retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { | ||
| 56 | const actorImagesInfo = processedImages.map(({ imageName, imageSize }) => ({ | ||
| 57 | name: imageName, | ||
| 58 | fileUrl: null, | ||
| 59 | height: imageSize.height, | ||
| 60 | width: imageSize.width, | ||
| 61 | onDisk: true | ||
| 62 | })) | ||
| 63 | |||
| 64 | const updatedActor = await updateActorImages(accountOrChannel.Actor, type, actorImagesInfo, t) | ||
| 65 | await updatedActor.save({ transaction: t }) | ||
| 66 | |||
| 67 | await sendUpdateActor(accountOrChannel, t) | ||
| 68 | |||
| 69 | return type === ActorImageType.AVATAR | ||
| 70 | ? updatedActor.Avatars | ||
| 71 | : updatedActor.Banners | ||
| 72 | })) | ||
| 73 | } | ||
| 74 | |||
| 75 | export async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) { | ||
| 76 | return retryTransactionWrapper(() => { | ||
| 77 | return sequelizeTypescript.transaction(async t => { | ||
| 78 | const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t) | ||
| 79 | await updatedActor.save({ transaction: t }) | ||
| 80 | |||
| 81 | await sendUpdateActor(accountOrChannel, t) | ||
| 82 | |||
| 83 | return updatedActor.Avatars | ||
| 84 | }) | ||
| 85 | }) | ||
| 86 | } | ||
| 87 | |||
| 88 | // --------------------------------------------------------------------------- | ||
| 89 | |||
| 90 | export async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) { | ||
| 91 | let actor = await ActorModel.loadLocalByName(baseActorName, transaction) | ||
| 92 | if (!actor) return baseActorName | ||
| 93 | |||
| 94 | for (let i = 1; i < 30; i++) { | ||
| 95 | const name = `${baseActorName}-${i}` | ||
| 96 | |||
| 97 | actor = await ActorModel.loadLocalByName(name, transaction) | ||
| 98 | if (!actor) return name | ||
| 99 | } | ||
| 100 | |||
| 101 | throw new Error('Cannot find available actor local name (too much iterations).') | ||
| 102 | } | ||
diff --git a/server/lib/model-loaders/actor.ts b/server/lib/model-loaders/actor.ts deleted file mode 100644 index 1355d8ee2..000000000 --- a/server/lib/model-loaders/actor.ts +++ /dev/null | |||
| @@ -1,17 +0,0 @@ | |||
| 1 | |||
| 2 | import { ActorModel } from '../../models/actor/actor' | ||
| 3 | import { MActorAccountChannelId, MActorFull } from '../../types/models' | ||
| 4 | |||
| 5 | type ActorLoadByUrlType = 'all' | 'association-ids' | ||
| 6 | |||
| 7 | function loadActorByUrl (url: string, fetchType: ActorLoadByUrlType): Promise<MActorFull | MActorAccountChannelId> { | ||
| 8 | if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url) | ||
| 9 | |||
| 10 | if (fetchType === 'association-ids') return ActorModel.loadByUrl(url) | ||
| 11 | } | ||
| 12 | |||
| 13 | export { | ||
| 14 | ActorLoadByUrlType, | ||
| 15 | |||
| 16 | loadActorByUrl | ||
| 17 | } | ||
diff --git a/server/lib/model-loaders/index.ts b/server/lib/model-loaders/index.ts deleted file mode 100644 index 9e5152cb2..000000000 --- a/server/lib/model-loaders/index.ts +++ /dev/null | |||
| @@ -1,2 +0,0 @@ | |||
| 1 | export * from './actor' | ||
| 2 | export * from './video' | ||
diff --git a/server/lib/model-loaders/video.ts b/server/lib/model-loaders/video.ts deleted file mode 100644 index 91057d405..000000000 --- a/server/lib/model-loaders/video.ts +++ /dev/null | |||
| @@ -1,66 +0,0 @@ | |||
| 1 | import { VideoModel } from '@server/models/video/video' | ||
| 2 | import { | ||
| 3 | MVideoAccountLightBlacklistAllFiles, | ||
| 4 | MVideoFormattableDetails, | ||
| 5 | MVideoFullLight, | ||
| 6 | MVideoId, | ||
| 7 | MVideoImmutable, | ||
| 8 | MVideoThumbnail | ||
| 9 | } from '@server/types/models' | ||
| 10 | |||
| 11 | type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes' | ||
| 12 | |||
| 13 | function loadVideo (id: number | string, fetchType: 'for-api', userId?: number): Promise<MVideoFormattableDetails> | ||
| 14 | function loadVideo (id: number | string, fetchType: 'all', userId?: number): Promise<MVideoFullLight> | ||
| 15 | function loadVideo (id: number | string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable> | ||
| 16 | function loadVideo (id: number | string, fetchType: 'only-video', userId?: number): Promise<MVideoThumbnail> | ||
| 17 | function loadVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Promise<MVideoId> | ||
| 18 | function loadVideo ( | ||
| 19 | id: number | string, | ||
| 20 | fetchType: VideoLoadType, | ||
| 21 | userId?: number | ||
| 22 | ): Promise<MVideoFullLight | MVideoThumbnail | MVideoId | MVideoImmutable> | ||
| 23 | function loadVideo ( | ||
| 24 | id: number | string, | ||
| 25 | fetchType: VideoLoadType, | ||
| 26 | userId?: number | ||
| 27 | ): Promise<MVideoFullLight | MVideoThumbnail | MVideoId | MVideoImmutable> { | ||
| 28 | |||
| 29 | if (fetchType === 'for-api') return VideoModel.loadForGetAPI({ id, userId }) | ||
| 30 | |||
| 31 | if (fetchType === 'all') return VideoModel.loadFull(id, undefined, userId) | ||
| 32 | |||
| 33 | if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id) | ||
| 34 | |||
| 35 | if (fetchType === 'only-video') return VideoModel.load(id) | ||
| 36 | |||
| 37 | if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) | ||
| 38 | } | ||
| 39 | |||
| 40 | type VideoLoadByUrlType = 'all' | 'only-video' | 'only-immutable-attributes' | ||
| 41 | |||
| 42 | function loadVideoByUrl (url: string, fetchType: 'all'): Promise<MVideoAccountLightBlacklistAllFiles> | ||
| 43 | function loadVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable> | ||
| 44 | function loadVideoByUrl (url: string, fetchType: 'only-video'): Promise<MVideoThumbnail> | ||
| 45 | function loadVideoByUrl ( | ||
| 46 | url: string, | ||
| 47 | fetchType: VideoLoadByUrlType | ||
| 48 | ): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> | ||
| 49 | function loadVideoByUrl ( | ||
| 50 | url: string, | ||
| 51 | fetchType: VideoLoadByUrlType | ||
| 52 | ): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> { | ||
| 53 | if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) | ||
| 54 | |||
| 55 | if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url) | ||
| 56 | |||
| 57 | if (fetchType === 'only-video') return VideoModel.loadByUrl(url) | ||
| 58 | } | ||
| 59 | |||
| 60 | export { | ||
| 61 | VideoLoadType, | ||
| 62 | VideoLoadByUrlType, | ||
| 63 | |||
| 64 | loadVideo, | ||
| 65 | loadVideoByUrl | ||
| 66 | } | ||
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts deleted file mode 100644 index db8284872..000000000 --- a/server/lib/moderation.ts +++ /dev/null | |||
| @@ -1,258 +0,0 @@ | |||
| 1 | import express, { VideoUploadFile } from 'express' | ||
| 2 | import { PathLike } from 'fs-extra' | ||
| 3 | import { Transaction } from 'sequelize/types' | ||
| 4 | import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' | ||
| 5 | import { afterCommitIfTransaction } from '@server/helpers/database-utils' | ||
| 6 | import { logger } from '@server/helpers/logger' | ||
| 7 | import { AbuseModel } from '@server/models/abuse/abuse' | ||
| 8 | import { VideoAbuseModel } from '@server/models/abuse/video-abuse' | ||
| 9 | import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse' | ||
| 10 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 11 | import { FilteredModelAttributes } from '@server/types' | ||
| 12 | import { | ||
| 13 | MAbuseFull, | ||
| 14 | MAccountDefault, | ||
| 15 | MAccountLight, | ||
| 16 | MComment, | ||
| 17 | MCommentAbuseAccountVideo, | ||
| 18 | MCommentOwnerVideo, | ||
| 19 | MUser, | ||
| 20 | MVideoAbuseVideoFull, | ||
| 21 | MVideoAccountLightBlacklistAllFiles | ||
| 22 | } from '@server/types/models' | ||
| 23 | import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' | ||
| 24 | import { VideoCommentCreate } from '../../shared/models/videos/comment' | ||
| 25 | import { UserModel } from '../models/user/user' | ||
| 26 | import { VideoModel } from '../models/video/video' | ||
| 27 | import { VideoCommentModel } from '../models/video/video-comment' | ||
| 28 | import { sendAbuse } from './activitypub/send/send-flag' | ||
| 29 | import { Notifier } from './notifier' | ||
| 30 | |||
| 31 | export type AcceptResult = { | ||
| 32 | accepted: boolean | ||
| 33 | errorMessage?: string | ||
| 34 | } | ||
| 35 | |||
| 36 | // --------------------------------------------------------------------------- | ||
| 37 | |||
| 38 | // Stub function that can be filtered by plugins | ||
| 39 | function isLocalVideoFileAccepted (object: { | ||
| 40 | videoBody: VideoCreate | ||
| 41 | videoFile: VideoUploadFile | ||
| 42 | user: UserModel | ||
| 43 | }): AcceptResult { | ||
| 44 | return { accepted: true } | ||
| 45 | } | ||
| 46 | |||
| 47 | // --------------------------------------------------------------------------- | ||
| 48 | |||
| 49 | // Stub function that can be filtered by plugins | ||
| 50 | function isLocalLiveVideoAccepted (object: { | ||
| 51 | liveVideoBody: LiveVideoCreate | ||
| 52 | user: UserModel | ||
| 53 | }): AcceptResult { | ||
| 54 | return { accepted: true } | ||
| 55 | } | ||
| 56 | |||
| 57 | // --------------------------------------------------------------------------- | ||
| 58 | |||
| 59 | // Stub function that can be filtered by plugins | ||
| 60 | function isLocalVideoThreadAccepted (_object: { | ||
| 61 | req: express.Request | ||
| 62 | commentBody: VideoCommentCreate | ||
| 63 | video: VideoModel | ||
| 64 | user: UserModel | ||
| 65 | }): AcceptResult { | ||
| 66 | return { accepted: true } | ||
| 67 | } | ||
| 68 | |||
| 69 | // Stub function that can be filtered by plugins | ||
| 70 | function isLocalVideoCommentReplyAccepted (_object: { | ||
| 71 | req: express.Request | ||
| 72 | commentBody: VideoCommentCreate | ||
| 73 | parentComment: VideoCommentModel | ||
| 74 | video: VideoModel | ||
| 75 | user: UserModel | ||
| 76 | }): AcceptResult { | ||
| 77 | return { accepted: true } | ||
| 78 | } | ||
| 79 | |||
| 80 | // --------------------------------------------------------------------------- | ||
| 81 | |||
| 82 | // Stub function that can be filtered by plugins | ||
| 83 | function isRemoteVideoCommentAccepted (_object: { | ||
| 84 | comment: MComment | ||
| 85 | }): AcceptResult { | ||
| 86 | return { accepted: true } | ||
| 87 | } | ||
| 88 | |||
| 89 | // --------------------------------------------------------------------------- | ||
| 90 | |||
| 91 | // Stub function that can be filtered by plugins | ||
| 92 | function isPreImportVideoAccepted (object: { | ||
| 93 | videoImportBody: VideoImportCreate | ||
| 94 | user: MUser | ||
| 95 | }): AcceptResult { | ||
| 96 | return { accepted: true } | ||
| 97 | } | ||
| 98 | |||
| 99 | // Stub function that can be filtered by plugins | ||
| 100 | function isPostImportVideoAccepted (object: { | ||
| 101 | videoFilePath: PathLike | ||
| 102 | videoFile: VideoFileModel | ||
| 103 | user: MUser | ||
| 104 | }): AcceptResult { | ||
| 105 | return { accepted: true } | ||
| 106 | } | ||
| 107 | |||
| 108 | // --------------------------------------------------------------------------- | ||
| 109 | |||
| 110 | async function createVideoAbuse (options: { | ||
| 111 | baseAbuse: FilteredModelAttributes<AbuseModel> | ||
| 112 | videoInstance: MVideoAccountLightBlacklistAllFiles | ||
| 113 | startAt: number | ||
| 114 | endAt: number | ||
| 115 | transaction: Transaction | ||
| 116 | reporterAccount: MAccountDefault | ||
| 117 | skipNotification: boolean | ||
| 118 | }) { | ||
| 119 | const { baseAbuse, videoInstance, startAt, endAt, transaction, reporterAccount, skipNotification } = options | ||
| 120 | |||
| 121 | const associateFun = async (abuseInstance: MAbuseFull) => { | ||
| 122 | const videoAbuseInstance: MVideoAbuseVideoFull = await VideoAbuseModel.create({ | ||
| 123 | abuseId: abuseInstance.id, | ||
| 124 | videoId: videoInstance.id, | ||
| 125 | startAt, | ||
| 126 | endAt | ||
| 127 | }, { transaction }) | ||
| 128 | |||
| 129 | videoAbuseInstance.Video = videoInstance | ||
| 130 | abuseInstance.VideoAbuse = videoAbuseInstance | ||
| 131 | |||
| 132 | return { isOwned: videoInstance.isOwned() } | ||
| 133 | } | ||
| 134 | |||
| 135 | return createAbuse({ | ||
| 136 | base: baseAbuse, | ||
| 137 | reporterAccount, | ||
| 138 | flaggedAccount: videoInstance.VideoChannel.Account, | ||
| 139 | transaction, | ||
| 140 | skipNotification, | ||
| 141 | associateFun | ||
| 142 | }) | ||
| 143 | } | ||
| 144 | |||
| 145 | function createVideoCommentAbuse (options: { | ||
| 146 | baseAbuse: FilteredModelAttributes<AbuseModel> | ||
| 147 | commentInstance: MCommentOwnerVideo | ||
| 148 | transaction: Transaction | ||
| 149 | reporterAccount: MAccountDefault | ||
| 150 | skipNotification: boolean | ||
| 151 | }) { | ||
| 152 | const { baseAbuse, commentInstance, transaction, reporterAccount, skipNotification } = options | ||
| 153 | |||
| 154 | const associateFun = async (abuseInstance: MAbuseFull) => { | ||
| 155 | const commentAbuseInstance: MCommentAbuseAccountVideo = await VideoCommentAbuseModel.create({ | ||
| 156 | abuseId: abuseInstance.id, | ||
| 157 | videoCommentId: commentInstance.id | ||
| 158 | }, { transaction }) | ||
| 159 | |||
| 160 | commentAbuseInstance.VideoComment = commentInstance | ||
| 161 | abuseInstance.VideoCommentAbuse = commentAbuseInstance | ||
| 162 | |||
| 163 | return { isOwned: commentInstance.isOwned() } | ||
| 164 | } | ||
| 165 | |||
| 166 | return createAbuse({ | ||
| 167 | base: baseAbuse, | ||
| 168 | reporterAccount, | ||
| 169 | flaggedAccount: commentInstance.Account, | ||
| 170 | transaction, | ||
| 171 | skipNotification, | ||
| 172 | associateFun | ||
| 173 | }) | ||
| 174 | } | ||
| 175 | |||
| 176 | function createAccountAbuse (options: { | ||
| 177 | baseAbuse: FilteredModelAttributes<AbuseModel> | ||
| 178 | accountInstance: MAccountDefault | ||
| 179 | transaction: Transaction | ||
| 180 | reporterAccount: MAccountDefault | ||
| 181 | skipNotification: boolean | ||
| 182 | }) { | ||
| 183 | const { baseAbuse, accountInstance, transaction, reporterAccount, skipNotification } = options | ||
| 184 | |||
| 185 | const associateFun = () => { | ||
| 186 | return Promise.resolve({ isOwned: accountInstance.isOwned() }) | ||
| 187 | } | ||
| 188 | |||
| 189 | return createAbuse({ | ||
| 190 | base: baseAbuse, | ||
| 191 | reporterAccount, | ||
| 192 | flaggedAccount: accountInstance, | ||
| 193 | transaction, | ||
| 194 | skipNotification, | ||
| 195 | associateFun | ||
| 196 | }) | ||
| 197 | } | ||
| 198 | |||
| 199 | // --------------------------------------------------------------------------- | ||
| 200 | |||
| 201 | export { | ||
| 202 | isLocalLiveVideoAccepted, | ||
| 203 | |||
| 204 | isLocalVideoFileAccepted, | ||
| 205 | isLocalVideoThreadAccepted, | ||
| 206 | isRemoteVideoCommentAccepted, | ||
| 207 | isLocalVideoCommentReplyAccepted, | ||
| 208 | isPreImportVideoAccepted, | ||
| 209 | isPostImportVideoAccepted, | ||
| 210 | |||
| 211 | createAbuse, | ||
| 212 | createVideoAbuse, | ||
| 213 | createVideoCommentAbuse, | ||
| 214 | createAccountAbuse | ||
| 215 | } | ||
| 216 | |||
| 217 | // --------------------------------------------------------------------------- | ||
| 218 | |||
| 219 | async function createAbuse (options: { | ||
| 220 | base: FilteredModelAttributes<AbuseModel> | ||
| 221 | reporterAccount: MAccountDefault | ||
| 222 | flaggedAccount: MAccountLight | ||
| 223 | associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean }> | ||
| 224 | skipNotification: boolean | ||
| 225 | transaction: Transaction | ||
| 226 | }) { | ||
| 227 | const { base, reporterAccount, flaggedAccount, associateFun, transaction, skipNotification } = options | ||
| 228 | const auditLogger = auditLoggerFactory('abuse') | ||
| 229 | |||
| 230 | const abuseAttributes = Object.assign({}, base, { flaggedAccountId: flaggedAccount.id }) | ||
| 231 | const abuseInstance: MAbuseFull = await AbuseModel.create(abuseAttributes, { transaction }) | ||
| 232 | |||
| 233 | abuseInstance.ReporterAccount = reporterAccount | ||
| 234 | abuseInstance.FlaggedAccount = flaggedAccount | ||
| 235 | |||
| 236 | const { isOwned } = await associateFun(abuseInstance) | ||
| 237 | |||
| 238 | if (isOwned === false) { | ||
| 239 | sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction) | ||
| 240 | } | ||
| 241 | |||
| 242 | const abuseJSON = abuseInstance.toFormattedAdminJSON() | ||
| 243 | auditLogger.create(reporterAccount.Actor.getIdentifier(), new AbuseAuditView(abuseJSON)) | ||
| 244 | |||
| 245 | if (!skipNotification) { | ||
| 246 | afterCommitIfTransaction(transaction, () => { | ||
| 247 | Notifier.Instance.notifyOnNewAbuse({ | ||
| 248 | abuse: abuseJSON, | ||
| 249 | abuseInstance, | ||
| 250 | reporter: reporterAccount.Actor.getIdentifier() | ||
| 251 | }) | ||
| 252 | }) | ||
| 253 | } | ||
| 254 | |||
| 255 | logger.info('Abuse report %d created.', abuseInstance.id) | ||
| 256 | |||
| 257 | return abuseJSON | ||
| 258 | } | ||
diff --git a/server/lib/notifier/index.ts b/server/lib/notifier/index.ts deleted file mode 100644 index 5bc2f5f50..000000000 --- a/server/lib/notifier/index.ts +++ /dev/null | |||
| @@ -1 +0,0 @@ | |||
| 1 | export * from './notifier' | ||
diff --git a/server/lib/notifier/notifier.ts b/server/lib/notifier/notifier.ts deleted file mode 100644 index 920c55df0..000000000 --- a/server/lib/notifier/notifier.ts +++ /dev/null | |||
| @@ -1,284 +0,0 @@ | |||
| 1 | import { MRegistration, MUser, MUserDefault } from '@server/types/models/user' | ||
| 2 | import { MVideoBlacklistLightVideo, MVideoBlacklistVideo } from '@server/types/models/video/video-blacklist' | ||
| 3 | import { UserNotificationSettingValue } from '../../../shared/models/users' | ||
| 4 | import { logger } from '../../helpers/logger' | ||
| 5 | import { CONFIG } from '../../initializers/config' | ||
| 6 | import { MAbuseFull, MAbuseMessage, MActorFollowFull, MApplication, MPlugin } from '../../types/models' | ||
| 7 | import { MCommentOwnerVideo, MVideoAccountLight, MVideoFullLight } from '../../types/models/video' | ||
| 8 | import { JobQueue } from '../job-queue' | ||
| 9 | import { PeerTubeSocket } from '../peertube-socket' | ||
| 10 | import { Hooks } from '../plugins/hooks' | ||
| 11 | import { | ||
| 12 | AbstractNotification, | ||
| 13 | AbuseStateChangeForReporter, | ||
| 14 | AutoFollowForInstance, | ||
| 15 | CommentMention, | ||
| 16 | DirectRegistrationForModerators, | ||
| 17 | FollowForInstance, | ||
| 18 | FollowForUser, | ||
| 19 | ImportFinishedForOwner, | ||
| 20 | ImportFinishedForOwnerPayload, | ||
| 21 | NewAbuseForModerators, | ||
| 22 | NewAbuseMessageForModerators, | ||
| 23 | NewAbuseMessageForReporter, | ||
| 24 | NewAbusePayload, | ||
| 25 | NewAutoBlacklistForModerators, | ||
| 26 | NewBlacklistForOwner, | ||
| 27 | NewCommentForVideoOwner, | ||
| 28 | NewPeerTubeVersionForAdmins, | ||
| 29 | NewPluginVersionForAdmins, | ||
| 30 | NewVideoForSubscribers, | ||
| 31 | OwnedPublicationAfterAutoUnblacklist, | ||
| 32 | OwnedPublicationAfterScheduleUpdate, | ||
| 33 | OwnedPublicationAfterTranscoding, | ||
| 34 | RegistrationRequestForModerators, | ||
| 35 | StudioEditionFinishedForOwner, | ||
| 36 | UnblacklistForOwner | ||
| 37 | } from './shared' | ||
| 38 | |||
| 39 | class Notifier { | ||
| 40 | |||
| 41 | private readonly notificationModels = { | ||
| 42 | newVideo: [ NewVideoForSubscribers ], | ||
| 43 | publicationAfterTranscoding: [ OwnedPublicationAfterTranscoding ], | ||
| 44 | publicationAfterScheduleUpdate: [ OwnedPublicationAfterScheduleUpdate ], | ||
| 45 | publicationAfterAutoUnblacklist: [ OwnedPublicationAfterAutoUnblacklist ], | ||
| 46 | newComment: [ CommentMention, NewCommentForVideoOwner ], | ||
| 47 | newAbuse: [ NewAbuseForModerators ], | ||
| 48 | newBlacklist: [ NewBlacklistForOwner ], | ||
| 49 | unblacklist: [ UnblacklistForOwner ], | ||
| 50 | importFinished: [ ImportFinishedForOwner ], | ||
| 51 | directRegistration: [ DirectRegistrationForModerators ], | ||
| 52 | registrationRequest: [ RegistrationRequestForModerators ], | ||
| 53 | userFollow: [ FollowForUser ], | ||
| 54 | instanceFollow: [ FollowForInstance ], | ||
| 55 | autoInstanceFollow: [ AutoFollowForInstance ], | ||
| 56 | newAutoBlacklist: [ NewAutoBlacklistForModerators ], | ||
| 57 | abuseStateChange: [ AbuseStateChangeForReporter ], | ||
| 58 | newAbuseMessage: [ NewAbuseMessageForReporter, NewAbuseMessageForModerators ], | ||
| 59 | newPeertubeVersion: [ NewPeerTubeVersionForAdmins ], | ||
| 60 | newPluginVersion: [ NewPluginVersionForAdmins ], | ||
| 61 | videoStudioEditionFinished: [ StudioEditionFinishedForOwner ] | ||
| 62 | } | ||
| 63 | |||
| 64 | private static instance: Notifier | ||
| 65 | |||
| 66 | private constructor () { | ||
| 67 | } | ||
| 68 | |||
| 69 | notifyOnNewVideoIfNeeded (video: MVideoAccountLight): void { | ||
| 70 | const models = this.notificationModels.newVideo | ||
| 71 | |||
| 72 | this.sendNotifications(models, video) | ||
| 73 | .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) | ||
| 74 | } | ||
| 75 | |||
| 76 | notifyOnVideoPublishedAfterTranscoding (video: MVideoFullLight): void { | ||
| 77 | const models = this.notificationModels.publicationAfterTranscoding | ||
| 78 | |||
| 79 | this.sendNotifications(models, video) | ||
| 80 | .catch(err => logger.error('Cannot notify owner that its video %s has been published after transcoding.', video.url, { err })) | ||
| 81 | } | ||
| 82 | |||
| 83 | notifyOnVideoPublishedAfterScheduledUpdate (video: MVideoFullLight): void { | ||
| 84 | const models = this.notificationModels.publicationAfterScheduleUpdate | ||
| 85 | |||
| 86 | this.sendNotifications(models, video) | ||
| 87 | .catch(err => logger.error('Cannot notify owner that its video %s has been published after scheduled update.', video.url, { err })) | ||
| 88 | } | ||
| 89 | |||
| 90 | notifyOnVideoPublishedAfterRemovedFromAutoBlacklist (video: MVideoFullLight): void { | ||
| 91 | const models = this.notificationModels.publicationAfterAutoUnblacklist | ||
| 92 | |||
| 93 | this.sendNotifications(models, video) | ||
| 94 | .catch(err => { | ||
| 95 | logger.error('Cannot notify owner that its video %s has been published after removed from auto-blacklist.', video.url, { err }) | ||
| 96 | }) | ||
| 97 | } | ||
| 98 | |||
| 99 | notifyOnNewComment (comment: MCommentOwnerVideo): void { | ||
| 100 | const models = this.notificationModels.newComment | ||
| 101 | |||
| 102 | this.sendNotifications(models, comment) | ||
| 103 | .catch(err => logger.error('Cannot notify of new comment.', comment.url, { err })) | ||
| 104 | } | ||
| 105 | |||
| 106 | notifyOnNewAbuse (payload: NewAbusePayload): void { | ||
| 107 | const models = this.notificationModels.newAbuse | ||
| 108 | |||
| 109 | this.sendNotifications(models, payload) | ||
| 110 | .catch(err => logger.error('Cannot notify of new abuse %d.', payload.abuseInstance.id, { err })) | ||
| 111 | } | ||
| 112 | |||
| 113 | notifyOnVideoAutoBlacklist (videoBlacklist: MVideoBlacklistLightVideo): void { | ||
| 114 | const models = this.notificationModels.newAutoBlacklist | ||
| 115 | |||
| 116 | this.sendNotifications(models, videoBlacklist) | ||
| 117 | .catch(err => logger.error('Cannot notify of auto-blacklist of video %s.', videoBlacklist.Video.url, { err })) | ||
| 118 | } | ||
| 119 | |||
| 120 | notifyOnVideoBlacklist (videoBlacklist: MVideoBlacklistVideo): void { | ||
| 121 | const models = this.notificationModels.newBlacklist | ||
| 122 | |||
| 123 | this.sendNotifications(models, videoBlacklist) | ||
| 124 | .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) | ||
| 125 | } | ||
| 126 | |||
| 127 | notifyOnVideoUnblacklist (video: MVideoFullLight): void { | ||
| 128 | const models = this.notificationModels.unblacklist | ||
| 129 | |||
| 130 | this.sendNotifications(models, video) | ||
| 131 | .catch(err => logger.error('Cannot notify video owner of unblacklist of %s.', video.url, { err })) | ||
| 132 | } | ||
| 133 | |||
| 134 | notifyOnFinishedVideoImport (payload: ImportFinishedForOwnerPayload): void { | ||
| 135 | const models = this.notificationModels.importFinished | ||
| 136 | |||
| 137 | this.sendNotifications(models, payload) | ||
| 138 | .catch(err => { | ||
| 139 | logger.error('Cannot notify owner that its video import %s is finished.', payload.videoImport.getTargetIdentifier(), { err }) | ||
| 140 | }) | ||
| 141 | } | ||
| 142 | |||
| 143 | notifyOnNewDirectRegistration (user: MUserDefault): void { | ||
| 144 | const models = this.notificationModels.directRegistration | ||
| 145 | |||
| 146 | this.sendNotifications(models, user) | ||
| 147 | .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) | ||
| 148 | } | ||
| 149 | |||
| 150 | notifyOnNewRegistrationRequest (registration: MRegistration): void { | ||
| 151 | const models = this.notificationModels.registrationRequest | ||
| 152 | |||
| 153 | this.sendNotifications(models, registration) | ||
| 154 | .catch(err => logger.error('Cannot notify moderators of new registration request (%s).', registration.username, { err })) | ||
| 155 | } | ||
| 156 | |||
| 157 | notifyOfNewUserFollow (actorFollow: MActorFollowFull): void { | ||
| 158 | const models = this.notificationModels.userFollow | ||
| 159 | |||
| 160 | this.sendNotifications(models, actorFollow) | ||
| 161 | .catch(err => { | ||
| 162 | logger.error( | ||
| 163 | 'Cannot notify owner of channel %s of a new follow by %s.', | ||
| 164 | actorFollow.ActorFollowing.VideoChannel.getDisplayName(), | ||
| 165 | actorFollow.ActorFollower.Account.getDisplayName(), | ||
| 166 | { err } | ||
| 167 | ) | ||
| 168 | }) | ||
| 169 | } | ||
| 170 | |||
| 171 | notifyOfNewInstanceFollow (actorFollow: MActorFollowFull): void { | ||
| 172 | const models = this.notificationModels.instanceFollow | ||
| 173 | |||
| 174 | this.sendNotifications(models, actorFollow) | ||
| 175 | .catch(err => logger.error('Cannot notify administrators of new follower %s.', actorFollow.ActorFollower.url, { err })) | ||
| 176 | } | ||
| 177 | |||
| 178 | notifyOfAutoInstanceFollowing (actorFollow: MActorFollowFull): void { | ||
| 179 | const models = this.notificationModels.autoInstanceFollow | ||
| 180 | |||
| 181 | this.sendNotifications(models, actorFollow) | ||
| 182 | .catch(err => logger.error('Cannot notify administrators of auto instance following %s.', actorFollow.ActorFollowing.url, { err })) | ||
| 183 | } | ||
| 184 | |||
| 185 | notifyOnAbuseStateChange (abuse: MAbuseFull): void { | ||
| 186 | const models = this.notificationModels.abuseStateChange | ||
| 187 | |||
| 188 | this.sendNotifications(models, abuse) | ||
| 189 | .catch(err => logger.error('Cannot notify of abuse %d state change.', abuse.id, { err })) | ||
| 190 | } | ||
| 191 | |||
| 192 | notifyOnAbuseMessage (abuse: MAbuseFull, message: MAbuseMessage): void { | ||
| 193 | const models = this.notificationModels.newAbuseMessage | ||
| 194 | |||
| 195 | this.sendNotifications(models, { abuse, message }) | ||
| 196 | .catch(err => logger.error('Cannot notify on new abuse %d message.', abuse.id, { err })) | ||
| 197 | } | ||
| 198 | |||
| 199 | notifyOfNewPeerTubeVersion (application: MApplication, latestVersion: string) { | ||
| 200 | const models = this.notificationModels.newPeertubeVersion | ||
| 201 | |||
| 202 | this.sendNotifications(models, { application, latestVersion }) | ||
| 203 | .catch(err => logger.error('Cannot notify on new PeerTubeb version %s.', latestVersion, { err })) | ||
| 204 | } | ||
| 205 | |||
| 206 | notifyOfNewPluginVersion (plugin: MPlugin) { | ||
| 207 | const models = this.notificationModels.newPluginVersion | ||
| 208 | |||
| 209 | this.sendNotifications(models, plugin) | ||
| 210 | .catch(err => logger.error('Cannot notify on new plugin version %s.', plugin.name, { err })) | ||
| 211 | } | ||
| 212 | |||
| 213 | notifyOfFinishedVideoStudioEdition (video: MVideoFullLight) { | ||
| 214 | const models = this.notificationModels.videoStudioEditionFinished | ||
| 215 | |||
| 216 | this.sendNotifications(models, video) | ||
| 217 | .catch(err => logger.error('Cannot notify on finished studio edition %s.', video.url, { err })) | ||
| 218 | } | ||
| 219 | |||
| 220 | private async notify <T> (object: AbstractNotification<T>) { | ||
| 221 | await object.prepare() | ||
| 222 | |||
| 223 | const users = object.getTargetUsers() | ||
| 224 | |||
| 225 | if (users.length === 0) return | ||
| 226 | if (await object.isDisabled()) return | ||
| 227 | |||
| 228 | object.log() | ||
| 229 | |||
| 230 | const toEmails: string[] = [] | ||
| 231 | |||
| 232 | for (const user of users) { | ||
| 233 | const setting = object.getSetting(user) | ||
| 234 | |||
| 235 | const webNotificationEnabled = this.isWebNotificationEnabled(setting) | ||
| 236 | const emailNotificationEnabled = this.isEmailEnabled(user, setting) | ||
| 237 | const notification = object.createNotification(user) | ||
| 238 | |||
| 239 | if (webNotificationEnabled) { | ||
| 240 | await notification.save() | ||
| 241 | |||
| 242 | PeerTubeSocket.Instance.sendNotification(user.id, notification) | ||
| 243 | } | ||
| 244 | |||
| 245 | if (emailNotificationEnabled) { | ||
| 246 | toEmails.push(user.email) | ||
| 247 | } | ||
| 248 | |||
| 249 | Hooks.runAction('action:notifier.notification.created', { webNotificationEnabled, emailNotificationEnabled, user, notification }) | ||
| 250 | } | ||
| 251 | |||
| 252 | for (const to of toEmails) { | ||
| 253 | const payload = await object.createEmail(to) | ||
| 254 | JobQueue.Instance.createJobAsync({ type: 'email', payload }) | ||
| 255 | } | ||
| 256 | } | ||
| 257 | |||
| 258 | private isEmailEnabled (user: MUser, value: UserNotificationSettingValue) { | ||
| 259 | if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified === false) return false | ||
| 260 | |||
| 261 | return value & UserNotificationSettingValue.EMAIL | ||
| 262 | } | ||
| 263 | |||
| 264 | private isWebNotificationEnabled (value: UserNotificationSettingValue) { | ||
| 265 | return value & UserNotificationSettingValue.WEB | ||
| 266 | } | ||
| 267 | |||
| 268 | private async sendNotifications <T> (models: (new (payload: T) => AbstractNotification<T>)[], payload: T) { | ||
| 269 | for (const model of models) { | ||
| 270 | // eslint-disable-next-line new-cap | ||
| 271 | await this.notify(new model(payload)) | ||
| 272 | } | ||
| 273 | } | ||
| 274 | |||
| 275 | static get Instance () { | ||
| 276 | return this.instance || (this.instance = new this()) | ||
| 277 | } | ||
| 278 | } | ||
| 279 | |||
| 280 | // --------------------------------------------------------------------------- | ||
| 281 | |||
| 282 | export { | ||
| 283 | Notifier | ||
| 284 | } | ||
diff --git a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts deleted file mode 100644 index 1dc1ccfc2..000000000 --- a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts +++ /dev/null | |||
| @@ -1,67 +0,0 @@ | |||
| 1 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 2 | import { AccountModel } from '@server/models/account/account' | ||
| 3 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 4 | import { MAbuseFull, MAbuseMessage, MAccountDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 5 | import { UserNotificationType } from '@shared/models' | ||
| 6 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 7 | |||
| 8 | type NewAbuseMessagePayload = { | ||
| 9 | abuse: MAbuseFull | ||
| 10 | message: MAbuseMessage | ||
| 11 | } | ||
| 12 | |||
| 13 | export abstract class AbstractNewAbuseMessage extends AbstractNotification <NewAbuseMessagePayload> { | ||
| 14 | protected messageAccount: MAccountDefault | ||
| 15 | |||
| 16 | async loadMessageAccount () { | ||
| 17 | this.messageAccount = await AccountModel.load(this.message.accountId) | ||
| 18 | } | ||
| 19 | |||
| 20 | getSetting (user: MUserWithNotificationSetting) { | ||
| 21 | return user.NotificationSetting.abuseNewMessage | ||
| 22 | } | ||
| 23 | |||
| 24 | createNotification (user: MUserWithNotificationSetting) { | ||
| 25 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 26 | type: UserNotificationType.ABUSE_NEW_MESSAGE, | ||
| 27 | userId: user.id, | ||
| 28 | abuseId: this.abuse.id | ||
| 29 | }) | ||
| 30 | notification.Abuse = this.abuse | ||
| 31 | |||
| 32 | return notification | ||
| 33 | } | ||
| 34 | |||
| 35 | protected createEmailFor (to: string, target: 'moderator' | 'reporter') { | ||
| 36 | const text = 'New message on report #' + this.abuse.id | ||
| 37 | const abuseUrl = target === 'moderator' | ||
| 38 | ? WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.abuse.id | ||
| 39 | : WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id | ||
| 40 | |||
| 41 | const action = { | ||
| 42 | text: 'View report #' + this.abuse.id, | ||
| 43 | url: abuseUrl | ||
| 44 | } | ||
| 45 | |||
| 46 | return { | ||
| 47 | template: 'abuse-new-message', | ||
| 48 | to, | ||
| 49 | subject: text, | ||
| 50 | locals: { | ||
| 51 | abuseId: this.abuse.id, | ||
| 52 | abuseUrl: action.url, | ||
| 53 | messageAccountName: this.messageAccount.getDisplayName(), | ||
| 54 | messageText: this.message.message, | ||
| 55 | action | ||
| 56 | } | ||
| 57 | } | ||
| 58 | } | ||
| 59 | |||
| 60 | protected get abuse () { | ||
| 61 | return this.payload.abuse | ||
| 62 | } | ||
| 63 | |||
| 64 | protected get message () { | ||
| 65 | return this.payload.message | ||
| 66 | } | ||
| 67 | } | ||
diff --git a/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts b/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts deleted file mode 100644 index 97e896c6a..000000000 --- a/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts +++ /dev/null | |||
| @@ -1,74 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { getAbuseTargetUrl } from '@server/lib/activitypub/url' | ||
| 4 | import { UserModel } from '@server/models/user/user' | ||
| 5 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 6 | import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 7 | import { AbuseState, UserNotificationType } from '@shared/models' | ||
| 8 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 9 | |||
| 10 | export class AbuseStateChangeForReporter extends AbstractNotification <MAbuseFull> { | ||
| 11 | |||
| 12 | private user: MUserDefault | ||
| 13 | |||
| 14 | async prepare () { | ||
| 15 | const reporter = this.abuse.ReporterAccount | ||
| 16 | if (reporter.isOwned() !== true) return | ||
| 17 | |||
| 18 | this.user = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) | ||
| 19 | } | ||
| 20 | |||
| 21 | log () { | ||
| 22 | logger.info('Notifying reporter of abuse % of state change.', getAbuseTargetUrl(this.abuse)) | ||
| 23 | } | ||
| 24 | |||
| 25 | getSetting (user: MUserWithNotificationSetting) { | ||
| 26 | return user.NotificationSetting.abuseStateChange | ||
| 27 | } | ||
| 28 | |||
| 29 | getTargetUsers () { | ||
| 30 | if (!this.user) return [] | ||
| 31 | |||
| 32 | return [ this.user ] | ||
| 33 | } | ||
| 34 | |||
| 35 | createNotification (user: MUserWithNotificationSetting) { | ||
| 36 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 37 | type: UserNotificationType.ABUSE_STATE_CHANGE, | ||
| 38 | userId: user.id, | ||
| 39 | abuseId: this.abuse.id | ||
| 40 | }) | ||
| 41 | notification.Abuse = this.abuse | ||
| 42 | |||
| 43 | return notification | ||
| 44 | } | ||
| 45 | |||
| 46 | createEmail (to: string) { | ||
| 47 | const text = this.abuse.state === AbuseState.ACCEPTED | ||
| 48 | ? 'Report #' + this.abuse.id + ' has been accepted' | ||
| 49 | : 'Report #' + this.abuse.id + ' has been rejected' | ||
| 50 | |||
| 51 | const abuseUrl = WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id | ||
| 52 | |||
| 53 | const action = { | ||
| 54 | text: 'View report #' + this.abuse.id, | ||
| 55 | url: abuseUrl | ||
| 56 | } | ||
| 57 | |||
| 58 | return { | ||
| 59 | template: 'abuse-state-change', | ||
| 60 | to, | ||
| 61 | subject: text, | ||
| 62 | locals: { | ||
| 63 | action, | ||
| 64 | abuseId: this.abuse.id, | ||
| 65 | abuseUrl, | ||
| 66 | isAccepted: this.abuse.state === AbuseState.ACCEPTED | ||
| 67 | } | ||
| 68 | } | ||
| 69 | } | ||
| 70 | |||
| 71 | private get abuse () { | ||
| 72 | return this.payload | ||
| 73 | } | ||
| 74 | } | ||
diff --git a/server/lib/notifier/shared/abuse/index.ts b/server/lib/notifier/shared/abuse/index.ts deleted file mode 100644 index 7b54c5591..000000000 --- a/server/lib/notifier/shared/abuse/index.ts +++ /dev/null | |||
| @@ -1,4 +0,0 @@ | |||
| 1 | export * from './abuse-state-change-for-reporter' | ||
| 2 | export * from './new-abuse-for-moderators' | ||
| 3 | export * from './new-abuse-message-for-reporter' | ||
| 4 | export * from './new-abuse-message-for-moderators' | ||
diff --git a/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts b/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts deleted file mode 100644 index 7d86fb55f..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts +++ /dev/null | |||
| @@ -1,119 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { getAbuseTargetUrl } from '@server/lib/activitypub/url' | ||
| 4 | import { UserModel } from '@server/models/user/user' | ||
| 5 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 6 | import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 7 | import { UserAbuse, UserNotificationType, UserRight } from '@shared/models' | ||
| 8 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 9 | |||
| 10 | export type NewAbusePayload = { abuse: UserAbuse, abuseInstance: MAbuseFull, reporter: string } | ||
| 11 | |||
| 12 | export class NewAbuseForModerators extends AbstractNotification <NewAbusePayload> { | ||
| 13 | private moderators: MUserDefault[] | ||
| 14 | |||
| 15 | async prepare () { | ||
| 16 | this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) | ||
| 17 | } | ||
| 18 | |||
| 19 | log () { | ||
| 20 | logger.info('Notifying %s user/moderators of new abuse %s.', this.moderators.length, getAbuseTargetUrl(this.payload.abuseInstance)) | ||
| 21 | } | ||
| 22 | |||
| 23 | getSetting (user: MUserWithNotificationSetting) { | ||
| 24 | return user.NotificationSetting.abuseAsModerator | ||
| 25 | } | ||
| 26 | |||
| 27 | getTargetUsers () { | ||
| 28 | return this.moderators | ||
| 29 | } | ||
| 30 | |||
| 31 | createNotification (user: MUserWithNotificationSetting) { | ||
| 32 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 33 | type: UserNotificationType.NEW_ABUSE_FOR_MODERATORS, | ||
| 34 | userId: user.id, | ||
| 35 | abuseId: this.payload.abuseInstance.id | ||
| 36 | }) | ||
| 37 | notification.Abuse = this.payload.abuseInstance | ||
| 38 | |||
| 39 | return notification | ||
| 40 | } | ||
| 41 | |||
| 42 | createEmail (to: string) { | ||
| 43 | const abuseInstance = this.payload.abuseInstance | ||
| 44 | |||
| 45 | if (abuseInstance.VideoAbuse) return this.createVideoAbuseEmail(to) | ||
| 46 | if (abuseInstance.VideoCommentAbuse) return this.createCommentAbuseEmail(to) | ||
| 47 | |||
| 48 | return this.createAccountAbuseEmail(to) | ||
| 49 | } | ||
| 50 | |||
| 51 | private createVideoAbuseEmail (to: string) { | ||
| 52 | const video = this.payload.abuseInstance.VideoAbuse.Video | ||
| 53 | const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() | ||
| 54 | |||
| 55 | return { | ||
| 56 | template: 'video-abuse-new', | ||
| 57 | to, | ||
| 58 | subject: `New video abuse report from ${this.payload.reporter}`, | ||
| 59 | locals: { | ||
| 60 | videoUrl, | ||
| 61 | isLocal: video.remote === false, | ||
| 62 | videoCreatedAt: new Date(video.createdAt).toLocaleString(), | ||
| 63 | videoPublishedAt: new Date(video.publishedAt).toLocaleString(), | ||
| 64 | videoName: video.name, | ||
| 65 | reason: this.payload.abuse.reason, | ||
| 66 | videoChannel: this.payload.abuse.video.channel, | ||
| 67 | reporter: this.payload.reporter, | ||
| 68 | action: this.buildEmailAction() | ||
| 69 | } | ||
| 70 | } | ||
| 71 | } | ||
| 72 | |||
| 73 | private createCommentAbuseEmail (to: string) { | ||
| 74 | const comment = this.payload.abuseInstance.VideoCommentAbuse.VideoComment | ||
| 75 | const commentUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() + ';threadId=' + comment.getThreadId() | ||
| 76 | |||
| 77 | return { | ||
| 78 | template: 'video-comment-abuse-new', | ||
| 79 | to, | ||
| 80 | subject: `New comment abuse report from ${this.payload.reporter}`, | ||
| 81 | locals: { | ||
| 82 | commentUrl, | ||
| 83 | videoName: comment.Video.name, | ||
| 84 | isLocal: comment.isOwned(), | ||
| 85 | commentCreatedAt: new Date(comment.createdAt).toLocaleString(), | ||
| 86 | reason: this.payload.abuse.reason, | ||
| 87 | flaggedAccount: this.payload.abuseInstance.FlaggedAccount.getDisplayName(), | ||
| 88 | reporter: this.payload.reporter, | ||
| 89 | action: this.buildEmailAction() | ||
| 90 | } | ||
| 91 | } | ||
| 92 | } | ||
| 93 | |||
| 94 | private createAccountAbuseEmail (to: string) { | ||
| 95 | const account = this.payload.abuseInstance.FlaggedAccount | ||
| 96 | const accountUrl = account.getClientUrl() | ||
| 97 | |||
| 98 | return { | ||
| 99 | template: 'account-abuse-new', | ||
| 100 | to, | ||
| 101 | subject: `New account abuse report from ${this.payload.reporter}`, | ||
| 102 | locals: { | ||
| 103 | accountUrl, | ||
| 104 | accountDisplayName: account.getDisplayName(), | ||
| 105 | isLocal: account.isOwned(), | ||
| 106 | reason: this.payload.abuse.reason, | ||
| 107 | reporter: this.payload.reporter, | ||
| 108 | action: this.buildEmailAction() | ||
| 109 | } | ||
| 110 | } | ||
| 111 | } | ||
| 112 | |||
| 113 | private buildEmailAction () { | ||
| 114 | return { | ||
| 115 | text: 'View report #' + this.payload.abuseInstance.id, | ||
| 116 | url: WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.payload.abuseInstance.id | ||
| 117 | } | ||
| 118 | } | ||
| 119 | } | ||
diff --git a/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts b/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts deleted file mode 100644 index 9d0629690..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts +++ /dev/null | |||
| @@ -1,32 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { getAbuseTargetUrl } from '@server/lib/activitypub/url' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { MUserDefault } from '@server/types/models' | ||
| 5 | import { UserRight } from '@shared/models' | ||
| 6 | import { AbstractNewAbuseMessage } from './abstract-new-abuse-message' | ||
| 7 | |||
| 8 | export class NewAbuseMessageForModerators extends AbstractNewAbuseMessage { | ||
| 9 | private moderators: MUserDefault[] | ||
| 10 | |||
| 11 | async prepare () { | ||
| 12 | this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) | ||
| 13 | |||
| 14 | // Don't notify my own message | ||
| 15 | this.moderators = this.moderators.filter(m => m.Account.id !== this.message.accountId) | ||
| 16 | if (this.moderators.length === 0) return | ||
| 17 | |||
| 18 | await this.loadMessageAccount() | ||
| 19 | } | ||
| 20 | |||
| 21 | log () { | ||
| 22 | logger.info('Notifying moderators of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) | ||
| 23 | } | ||
| 24 | |||
| 25 | getTargetUsers () { | ||
| 26 | return this.moderators | ||
| 27 | } | ||
| 28 | |||
| 29 | createEmail (to: string) { | ||
| 30 | return this.createEmailFor(to, 'moderator') | ||
| 31 | } | ||
| 32 | } | ||
diff --git a/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts b/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts deleted file mode 100644 index c5bbb5447..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts +++ /dev/null | |||
| @@ -1,36 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { getAbuseTargetUrl } from '@server/lib/activitypub/url' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { MUserDefault } from '@server/types/models' | ||
| 5 | import { AbstractNewAbuseMessage } from './abstract-new-abuse-message' | ||
| 6 | |||
| 7 | export class NewAbuseMessageForReporter extends AbstractNewAbuseMessage { | ||
| 8 | private reporter: MUserDefault | ||
| 9 | |||
| 10 | async prepare () { | ||
| 11 | // Only notify our users | ||
| 12 | if (this.abuse.ReporterAccount.isOwned() !== true) return | ||
| 13 | |||
| 14 | await this.loadMessageAccount() | ||
| 15 | |||
| 16 | const reporter = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) | ||
| 17 | // Don't notify my own message | ||
| 18 | if (reporter.Account.id === this.message.accountId) return | ||
| 19 | |||
| 20 | this.reporter = reporter | ||
| 21 | } | ||
| 22 | |||
| 23 | log () { | ||
| 24 | logger.info('Notifying reporter of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) | ||
| 25 | } | ||
| 26 | |||
| 27 | getTargetUsers () { | ||
| 28 | if (!this.reporter) return [] | ||
| 29 | |||
| 30 | return [ this.reporter ] | ||
| 31 | } | ||
| 32 | |||
| 33 | createEmail (to: string) { | ||
| 34 | return this.createEmailFor(to, 'reporter') | ||
| 35 | } | ||
| 36 | } | ||
diff --git a/server/lib/notifier/shared/blacklist/index.ts b/server/lib/notifier/shared/blacklist/index.ts deleted file mode 100644 index 2f98d88ae..000000000 --- a/server/lib/notifier/shared/blacklist/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './new-auto-blacklist-for-moderators' | ||
| 2 | export * from './new-blacklist-for-owner' | ||
| 3 | export * from './unblacklist-for-owner' | ||
diff --git a/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts b/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts deleted file mode 100644 index ad2cc00ea..000000000 --- a/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts +++ /dev/null | |||
| @@ -1,60 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
| 6 | import { MUserDefault, MUserWithNotificationSetting, MVideoBlacklistLightVideo, UserNotificationModelForApi } from '@server/types/models' | ||
| 7 | import { UserNotificationType, UserRight } from '@shared/models' | ||
| 8 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 9 | |||
| 10 | export class NewAutoBlacklistForModerators extends AbstractNotification <MVideoBlacklistLightVideo> { | ||
| 11 | private moderators: MUserDefault[] | ||
| 12 | |||
| 13 | async prepare () { | ||
| 14 | this.moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_BLACKLIST) | ||
| 15 | } | ||
| 16 | |||
| 17 | log () { | ||
| 18 | logger.info('Notifying %s moderators of video auto-blacklist %s.', this.moderators.length, this.payload.Video.url) | ||
| 19 | } | ||
| 20 | |||
| 21 | getSetting (user: MUserWithNotificationSetting) { | ||
| 22 | return user.NotificationSetting.videoAutoBlacklistAsModerator | ||
| 23 | } | ||
| 24 | |||
| 25 | getTargetUsers () { | ||
| 26 | return this.moderators | ||
| 27 | } | ||
| 28 | |||
| 29 | createNotification (user: MUserWithNotificationSetting) { | ||
| 30 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 31 | type: UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS, | ||
| 32 | userId: user.id, | ||
| 33 | videoBlacklistId: this.payload.id | ||
| 34 | }) | ||
| 35 | notification.VideoBlacklist = this.payload | ||
| 36 | |||
| 37 | return notification | ||
| 38 | } | ||
| 39 | |||
| 40 | async createEmail (to: string) { | ||
| 41 | const videoAutoBlacklistUrl = WEBSERVER.URL + '/admin/moderation/video-auto-blacklist/list' | ||
| 42 | const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() | ||
| 43 | const channel = await VideoChannelModel.loadAndPopulateAccount(this.payload.Video.channelId) | ||
| 44 | |||
| 45 | return { | ||
| 46 | template: 'video-auto-blacklist-new', | ||
| 47 | to, | ||
| 48 | subject: 'A new video is pending moderation', | ||
| 49 | locals: { | ||
| 50 | channel: channel.toFormattedSummaryJSON(), | ||
| 51 | videoUrl, | ||
| 52 | videoName: this.payload.Video.name, | ||
| 53 | action: { | ||
| 54 | text: 'Review autoblacklist', | ||
| 55 | url: videoAutoBlacklistUrl | ||
| 56 | } | ||
| 57 | } | ||
| 58 | } | ||
| 59 | } | ||
| 60 | } | ||
diff --git a/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts b/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts deleted file mode 100644 index 342b69ec7..000000000 --- a/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 4 | import { UserModel } from '@server/models/user/user' | ||
| 5 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 6 | import { MUserDefault, MUserWithNotificationSetting, MVideoBlacklistVideo, UserNotificationModelForApi } from '@server/types/models' | ||
| 7 | import { UserNotificationType } from '@shared/models' | ||
| 8 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 9 | |||
| 10 | export class NewBlacklistForOwner extends AbstractNotification <MVideoBlacklistVideo> { | ||
| 11 | private user: MUserDefault | ||
| 12 | |||
| 13 | async prepare () { | ||
| 14 | this.user = await UserModel.loadByVideoId(this.payload.videoId) | ||
| 15 | } | ||
| 16 | |||
| 17 | log () { | ||
| 18 | logger.info('Notifying user %s that its video %s has been blacklisted.', this.user.username, this.payload.Video.url) | ||
| 19 | } | ||
| 20 | |||
| 21 | getSetting (user: MUserWithNotificationSetting) { | ||
| 22 | return user.NotificationSetting.blacklistOnMyVideo | ||
| 23 | } | ||
| 24 | |||
| 25 | getTargetUsers () { | ||
| 26 | if (!this.user) return [] | ||
| 27 | |||
| 28 | return [ this.user ] | ||
| 29 | } | ||
| 30 | |||
| 31 | createNotification (user: MUserWithNotificationSetting) { | ||
| 32 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 33 | type: UserNotificationType.BLACKLIST_ON_MY_VIDEO, | ||
| 34 | userId: user.id, | ||
| 35 | videoBlacklistId: this.payload.id | ||
| 36 | }) | ||
| 37 | notification.VideoBlacklist = this.payload | ||
| 38 | |||
| 39 | return notification | ||
| 40 | } | ||
| 41 | |||
| 42 | createEmail (to: string) { | ||
| 43 | const videoName = this.payload.Video.name | ||
| 44 | const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() | ||
| 45 | |||
| 46 | const reasonString = this.payload.reason ? ` for the following reason: ${this.payload.reason}` : '' | ||
| 47 | const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.INSTANCE.NAME} has been blacklisted${reasonString}.` | ||
| 48 | |||
| 49 | return { | ||
| 50 | to, | ||
| 51 | subject: `Video ${videoName} blacklisted`, | ||
| 52 | text: blockedString, | ||
| 53 | locals: { | ||
| 54 | title: 'Your video was blacklisted' | ||
| 55 | } | ||
| 56 | } | ||
| 57 | } | ||
| 58 | } | ||
diff --git a/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts b/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts deleted file mode 100644 index e6f90e23c..000000000 --- a/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts +++ /dev/null | |||
| @@ -1,55 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 4 | import { UserModel } from '@server/models/user/user' | ||
| 5 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 6 | import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' | ||
| 7 | import { UserNotificationType } from '@shared/models' | ||
| 8 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 9 | |||
| 10 | export class UnblacklistForOwner extends AbstractNotification <MVideoFullLight> { | ||
| 11 | private user: MUserDefault | ||
| 12 | |||
| 13 | async prepare () { | ||
| 14 | this.user = await UserModel.loadByVideoId(this.payload.id) | ||
| 15 | } | ||
| 16 | |||
| 17 | log () { | ||
| 18 | logger.info('Notifying user %s that its video %s has been unblacklisted.', this.user.username, this.payload.url) | ||
| 19 | } | ||
| 20 | |||
| 21 | getSetting (user: MUserWithNotificationSetting) { | ||
| 22 | return user.NotificationSetting.blacklistOnMyVideo | ||
| 23 | } | ||
| 24 | |||
| 25 | getTargetUsers () { | ||
| 26 | if (!this.user) return [] | ||
| 27 | |||
| 28 | return [ this.user ] | ||
| 29 | } | ||
| 30 | |||
| 31 | createNotification (user: MUserWithNotificationSetting) { | ||
| 32 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 33 | type: UserNotificationType.UNBLACKLIST_ON_MY_VIDEO, | ||
| 34 | userId: user.id, | ||
| 35 | videoId: this.payload.id | ||
| 36 | }) | ||
| 37 | notification.Video = this.payload | ||
| 38 | |||
| 39 | return notification | ||
| 40 | } | ||
| 41 | |||
| 42 | createEmail (to: string) { | ||
| 43 | const video = this.payload | ||
| 44 | const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() | ||
| 45 | |||
| 46 | return { | ||
| 47 | to, | ||
| 48 | subject: `Video ${video.name} unblacklisted`, | ||
| 49 | text: `Your video "${video.name}" (${videoUrl}) on ${CONFIG.INSTANCE.NAME} has been unblacklisted.`, | ||
| 50 | locals: { | ||
| 51 | title: 'Your video was unblacklisted' | ||
| 52 | } | ||
| 53 | } | ||
| 54 | } | ||
| 55 | } | ||
diff --git a/server/lib/notifier/shared/comment/comment-mention.ts b/server/lib/notifier/shared/comment/comment-mention.ts deleted file mode 100644 index 3074e97db..000000000 --- a/server/lib/notifier/shared/comment/comment-mention.ts +++ /dev/null | |||
| @@ -1,111 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { toSafeHtml } from '@server/helpers/markdown' | ||
| 3 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 4 | import { AccountBlocklistModel } from '@server/models/account/account-blocklist' | ||
| 5 | import { getServerActor } from '@server/models/application/application' | ||
| 6 | import { ServerBlocklistModel } from '@server/models/server/server-blocklist' | ||
| 7 | import { UserModel } from '@server/models/user/user' | ||
| 8 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 9 | import { | ||
| 10 | MCommentOwnerVideo, | ||
| 11 | MUserDefault, | ||
| 12 | MUserNotifSettingAccount, | ||
| 13 | MUserWithNotificationSetting, | ||
| 14 | UserNotificationModelForApi | ||
| 15 | } from '@server/types/models' | ||
| 16 | import { UserNotificationSettingValue, UserNotificationType } from '@shared/models' | ||
| 17 | import { AbstractNotification } from '../common' | ||
| 18 | |||
| 19 | export class CommentMention extends AbstractNotification <MCommentOwnerVideo, MUserNotifSettingAccount> { | ||
| 20 | private users: MUserDefault[] | ||
| 21 | |||
| 22 | private serverAccountId: number | ||
| 23 | |||
| 24 | private accountMutedHash: { [ id: number ]: boolean } | ||
| 25 | private instanceMutedHash: { [ id: number ]: boolean } | ||
| 26 | |||
| 27 | async prepare () { | ||
| 28 | const extractedUsernames = this.payload.extractMentions() | ||
| 29 | logger.debug( | ||
| 30 | 'Extracted %d username from comment %s.', extractedUsernames.length, this.payload.url, | ||
| 31 | { usernames: extractedUsernames, text: this.payload.text } | ||
| 32 | ) | ||
| 33 | |||
| 34 | this.users = await UserModel.listByUsernames(extractedUsernames) | ||
| 35 | |||
| 36 | if (this.payload.Video.isOwned()) { | ||
| 37 | const userException = await UserModel.loadByVideoId(this.payload.videoId) | ||
| 38 | this.users = this.users.filter(u => u.id !== userException.id) | ||
| 39 | } | ||
| 40 | |||
| 41 | // Don't notify if I mentioned myself | ||
| 42 | this.users = this.users.filter(u => u.Account.id !== this.payload.accountId) | ||
| 43 | |||
| 44 | if (this.users.length === 0) return | ||
| 45 | |||
| 46 | this.serverAccountId = (await getServerActor()).Account.id | ||
| 47 | |||
| 48 | const sourceAccounts = this.users.map(u => u.Account.id).concat([ this.serverAccountId ]) | ||
| 49 | |||
| 50 | this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, this.payload.accountId) | ||
| 51 | this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, this.payload.Account.Actor.serverId) | ||
| 52 | } | ||
| 53 | |||
| 54 | log () { | ||
| 55 | logger.info('Notifying %d users of new comment %s.', this.users.length, this.payload.url) | ||
| 56 | } | ||
| 57 | |||
| 58 | getSetting (user: MUserNotifSettingAccount) { | ||
| 59 | const accountId = user.Account.id | ||
| 60 | if ( | ||
| 61 | this.accountMutedHash[accountId] === true || this.instanceMutedHash[accountId] === true || | ||
| 62 | this.accountMutedHash[this.serverAccountId] === true || this.instanceMutedHash[this.serverAccountId] === true | ||
| 63 | ) { | ||
| 64 | return UserNotificationSettingValue.NONE | ||
| 65 | } | ||
| 66 | |||
| 67 | return user.NotificationSetting.commentMention | ||
| 68 | } | ||
| 69 | |||
| 70 | getTargetUsers () { | ||
| 71 | return this.users | ||
| 72 | } | ||
| 73 | |||
| 74 | createNotification (user: MUserWithNotificationSetting) { | ||
| 75 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 76 | type: UserNotificationType.COMMENT_MENTION, | ||
| 77 | userId: user.id, | ||
| 78 | commentId: this.payload.id | ||
| 79 | }) | ||
| 80 | notification.VideoComment = this.payload | ||
| 81 | |||
| 82 | return notification | ||
| 83 | } | ||
| 84 | |||
| 85 | createEmail (to: string) { | ||
| 86 | const comment = this.payload | ||
| 87 | |||
| 88 | const accountName = comment.Account.getDisplayName() | ||
| 89 | const video = comment.Video | ||
| 90 | const videoUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() | ||
| 91 | const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath() | ||
| 92 | const commentHtml = toSafeHtml(comment.text) | ||
| 93 | |||
| 94 | return { | ||
| 95 | template: 'video-comment-mention', | ||
| 96 | to, | ||
| 97 | subject: 'Mention on video ' + video.name, | ||
| 98 | locals: { | ||
| 99 | comment, | ||
| 100 | commentHtml, | ||
| 101 | video, | ||
| 102 | videoUrl, | ||
| 103 | accountName, | ||
| 104 | action: { | ||
| 105 | text: 'View comment', | ||
| 106 | url: commentUrl | ||
| 107 | } | ||
| 108 | } | ||
| 109 | } | ||
| 110 | } | ||
| 111 | } | ||
diff --git a/server/lib/notifier/shared/comment/index.ts b/server/lib/notifier/shared/comment/index.ts deleted file mode 100644 index ae01a9646..000000000 --- a/server/lib/notifier/shared/comment/index.ts +++ /dev/null | |||
| @@ -1,2 +0,0 @@ | |||
| 1 | export * from './comment-mention' | ||
| 2 | export * from './new-comment-for-video-owner' | ||
diff --git a/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts b/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts deleted file mode 100644 index 4f96439a3..000000000 --- a/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts +++ /dev/null | |||
| @@ -1,76 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { toSafeHtml } from '@server/helpers/markdown' | ||
| 3 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 4 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | ||
| 5 | import { UserModel } from '@server/models/user/user' | ||
| 6 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 7 | import { MCommentOwnerVideo, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 8 | import { UserNotificationType } from '@shared/models' | ||
| 9 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 10 | |||
| 11 | export class NewCommentForVideoOwner extends AbstractNotification <MCommentOwnerVideo> { | ||
| 12 | private user: MUserDefault | ||
| 13 | |||
| 14 | async prepare () { | ||
| 15 | this.user = await UserModel.loadByVideoId(this.payload.videoId) | ||
| 16 | } | ||
| 17 | |||
| 18 | log () { | ||
| 19 | logger.info('Notifying owner of a video %s of new comment %s.', this.user.username, this.payload.url) | ||
| 20 | } | ||
| 21 | |||
| 22 | isDisabled () { | ||
| 23 | if (this.payload.Video.isOwned() === false) return true | ||
| 24 | |||
| 25 | // Not our user or user comments its own video | ||
| 26 | if (!this.user || this.payload.Account.userId === this.user.id) return true | ||
| 27 | |||
| 28 | return isBlockedByServerOrAccount(this.payload.Account, this.user.Account) | ||
| 29 | } | ||
| 30 | |||
| 31 | getSetting (user: MUserWithNotificationSetting) { | ||
| 32 | return user.NotificationSetting.newCommentOnMyVideo | ||
| 33 | } | ||
| 34 | |||
| 35 | getTargetUsers () { | ||
| 36 | if (!this.user) return [] | ||
| 37 | |||
| 38 | return [ this.user ] | ||
| 39 | } | ||
| 40 | |||
| 41 | createNotification (user: MUserWithNotificationSetting) { | ||
| 42 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 43 | type: UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, | ||
| 44 | userId: user.id, | ||
| 45 | commentId: this.payload.id | ||
| 46 | }) | ||
| 47 | notification.VideoComment = this.payload | ||
| 48 | |||
| 49 | return notification | ||
| 50 | } | ||
| 51 | |||
| 52 | createEmail (to: string) { | ||
| 53 | const video = this.payload.Video | ||
| 54 | const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() | ||
| 55 | const commentUrl = WEBSERVER.URL + this.payload.getCommentStaticPath() | ||
| 56 | const commentHtml = toSafeHtml(this.payload.text) | ||
| 57 | |||
| 58 | return { | ||
| 59 | template: 'video-comment-new', | ||
| 60 | to, | ||
| 61 | subject: 'New comment on your video ' + video.name, | ||
| 62 | locals: { | ||
| 63 | accountName: this.payload.Account.getDisplayName(), | ||
| 64 | accountUrl: this.payload.Account.Actor.url, | ||
| 65 | comment: this.payload, | ||
| 66 | commentHtml, | ||
| 67 | video, | ||
| 68 | videoUrl, | ||
| 69 | action: { | ||
| 70 | text: 'View comment', | ||
| 71 | url: commentUrl | ||
| 72 | } | ||
| 73 | } | ||
| 74 | } | ||
| 75 | } | ||
| 76 | } | ||
diff --git a/server/lib/notifier/shared/common/abstract-notification.ts b/server/lib/notifier/shared/common/abstract-notification.ts deleted file mode 100644 index 79403611e..000000000 --- a/server/lib/notifier/shared/common/abstract-notification.ts +++ /dev/null | |||
| @@ -1,23 +0,0 @@ | |||
| 1 | import { MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 2 | import { EmailPayload, UserNotificationSettingValue } from '@shared/models' | ||
| 3 | |||
| 4 | export abstract class AbstractNotification <T, U = MUserWithNotificationSetting> { | ||
| 5 | |||
| 6 | constructor (protected readonly payload: T) { | ||
| 7 | |||
| 8 | } | ||
| 9 | |||
| 10 | abstract prepare (): Promise<void> | ||
| 11 | abstract log (): void | ||
| 12 | |||
| 13 | abstract getSetting (user: U): UserNotificationSettingValue | ||
| 14 | abstract getTargetUsers (): U[] | ||
| 15 | |||
| 16 | abstract createNotification (user: U): UserNotificationModelForApi | ||
| 17 | abstract createEmail (to: string): EmailPayload | Promise<EmailPayload> | ||
| 18 | |||
| 19 | isDisabled (): boolean | Promise<boolean> { | ||
| 20 | return false | ||
| 21 | } | ||
| 22 | |||
| 23 | } | ||
diff --git a/server/lib/notifier/shared/common/index.ts b/server/lib/notifier/shared/common/index.ts deleted file mode 100644 index 0b2570278..000000000 --- a/server/lib/notifier/shared/common/index.ts +++ /dev/null | |||
| @@ -1 +0,0 @@ | |||
| 1 | export * from './abstract-notification' | ||
diff --git a/server/lib/notifier/shared/follow/auto-follow-for-instance.ts b/server/lib/notifier/shared/follow/auto-follow-for-instance.ts deleted file mode 100644 index ab9747ba8..000000000 --- a/server/lib/notifier/shared/follow/auto-follow-for-instance.ts +++ /dev/null | |||
| @@ -1,51 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { UserModel } from '@server/models/user/user' | ||
| 3 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 4 | import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 5 | import { UserNotificationType, UserRight } from '@shared/models' | ||
| 6 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 7 | |||
| 8 | export class AutoFollowForInstance extends AbstractNotification <MActorFollowFull> { | ||
| 9 | private admins: MUserDefault[] | ||
| 10 | |||
| 11 | async prepare () { | ||
| 12 | this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) | ||
| 13 | } | ||
| 14 | |||
| 15 | log () { | ||
| 16 | logger.info('Notifying %d administrators of auto instance following: %s.', this.admins.length, this.actorFollow.ActorFollowing.url) | ||
| 17 | } | ||
| 18 | |||
| 19 | getSetting (user: MUserWithNotificationSetting) { | ||
| 20 | return user.NotificationSetting.autoInstanceFollowing | ||
| 21 | } | ||
| 22 | |||
| 23 | getTargetUsers () { | ||
| 24 | return this.admins | ||
| 25 | } | ||
| 26 | |||
| 27 | createNotification (user: MUserWithNotificationSetting) { | ||
| 28 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 29 | type: UserNotificationType.AUTO_INSTANCE_FOLLOWING, | ||
| 30 | userId: user.id, | ||
| 31 | actorFollowId: this.actorFollow.id | ||
| 32 | }) | ||
| 33 | notification.ActorFollow = this.actorFollow | ||
| 34 | |||
| 35 | return notification | ||
| 36 | } | ||
| 37 | |||
| 38 | createEmail (to: string) { | ||
| 39 | const instanceUrl = this.actorFollow.ActorFollowing.url | ||
| 40 | |||
| 41 | return { | ||
| 42 | to, | ||
| 43 | subject: 'Auto instance following', | ||
| 44 | text: `Your instance automatically followed a new instance: <a href="${instanceUrl}">${instanceUrl}</a>.` | ||
| 45 | } | ||
| 46 | } | ||
| 47 | |||
| 48 | private get actorFollow () { | ||
| 49 | return this.payload | ||
| 50 | } | ||
| 51 | } | ||
diff --git a/server/lib/notifier/shared/follow/follow-for-instance.ts b/server/lib/notifier/shared/follow/follow-for-instance.ts deleted file mode 100644 index 777a12ef4..000000000 --- a/server/lib/notifier/shared/follow/follow-for-instance.ts +++ /dev/null | |||
| @@ -1,68 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | ||
| 4 | import { UserModel } from '@server/models/user/user' | ||
| 5 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 6 | import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 7 | import { UserNotificationType, UserRight } from '@shared/models' | ||
| 8 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 9 | |||
| 10 | export class FollowForInstance extends AbstractNotification <MActorFollowFull> { | ||
| 11 | private admins: MUserDefault[] | ||
| 12 | |||
| 13 | async prepare () { | ||
| 14 | this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) | ||
| 15 | } | ||
| 16 | |||
| 17 | isDisabled () { | ||
| 18 | const follower = Object.assign(this.actorFollow.ActorFollower.Account, { Actor: this.actorFollow.ActorFollower }) | ||
| 19 | |||
| 20 | return isBlockedByServerOrAccount(follower) | ||
| 21 | } | ||
| 22 | |||
| 23 | log () { | ||
| 24 | logger.info('Notifying %d administrators of new instance follower: %s.', this.admins.length, this.actorFollow.ActorFollower.url) | ||
| 25 | } | ||
| 26 | |||
| 27 | getSetting (user: MUserWithNotificationSetting) { | ||
| 28 | return user.NotificationSetting.newInstanceFollower | ||
| 29 | } | ||
| 30 | |||
| 31 | getTargetUsers () { | ||
| 32 | return this.admins | ||
| 33 | } | ||
| 34 | |||
| 35 | createNotification (user: MUserWithNotificationSetting) { | ||
| 36 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 37 | type: UserNotificationType.NEW_INSTANCE_FOLLOWER, | ||
| 38 | userId: user.id, | ||
| 39 | actorFollowId: this.actorFollow.id | ||
| 40 | }) | ||
| 41 | notification.ActorFollow = this.actorFollow | ||
| 42 | |||
| 43 | return notification | ||
| 44 | } | ||
| 45 | |||
| 46 | createEmail (to: string) { | ||
| 47 | const awaitingApproval = this.actorFollow.state === 'pending' | ||
| 48 | ? ' awaiting manual approval.' | ||
| 49 | : '' | ||
| 50 | |||
| 51 | return { | ||
| 52 | to, | ||
| 53 | subject: 'New instance follower', | ||
| 54 | text: `Your instance has a new follower: ${this.actorFollow.ActorFollower.url}${awaitingApproval}.`, | ||
| 55 | locals: { | ||
| 56 | title: 'New instance follower', | ||
| 57 | action: { | ||
| 58 | text: 'Review followers', | ||
| 59 | url: WEBSERVER.URL + '/admin/follows/followers-list' | ||
| 60 | } | ||
| 61 | } | ||
| 62 | } | ||
| 63 | } | ||
| 64 | |||
| 65 | private get actorFollow () { | ||
| 66 | return this.payload | ||
| 67 | } | ||
| 68 | } | ||
diff --git a/server/lib/notifier/shared/follow/follow-for-user.ts b/server/lib/notifier/shared/follow/follow-for-user.ts deleted file mode 100644 index 697c82cdd..000000000 --- a/server/lib/notifier/shared/follow/follow-for-user.ts +++ /dev/null | |||
| @@ -1,82 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 6 | import { UserNotificationType } from '@shared/models' | ||
| 7 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 8 | |||
| 9 | export class FollowForUser extends AbstractNotification <MActorFollowFull> { | ||
| 10 | private followType: 'account' | 'channel' | ||
| 11 | private user: MUserDefault | ||
| 12 | |||
| 13 | async prepare () { | ||
| 14 | // Account follows one of our account? | ||
| 15 | this.followType = 'channel' | ||
| 16 | this.user = await UserModel.loadByChannelActorId(this.actorFollow.ActorFollowing.id) | ||
| 17 | |||
| 18 | // Account follows one of our channel? | ||
| 19 | if (!this.user) { | ||
| 20 | this.user = await UserModel.loadByAccountActorId(this.actorFollow.ActorFollowing.id) | ||
| 21 | this.followType = 'account' | ||
| 22 | } | ||
| 23 | } | ||
| 24 | |||
| 25 | async isDisabled () { | ||
| 26 | if (this.payload.ActorFollowing.isOwned() === false) return true | ||
| 27 | |||
| 28 | const followerAccount = this.actorFollow.ActorFollower.Account | ||
| 29 | const followerAccountWithActor = Object.assign(followerAccount, { Actor: this.actorFollow.ActorFollower }) | ||
| 30 | |||
| 31 | return isBlockedByServerOrAccount(followerAccountWithActor, this.user.Account) | ||
| 32 | } | ||
| 33 | |||
| 34 | log () { | ||
| 35 | logger.info('Notifying user %s of new follower: %s.', this.user.username, this.actorFollow.ActorFollower.Account.getDisplayName()) | ||
| 36 | } | ||
| 37 | |||
| 38 | getSetting (user: MUserWithNotificationSetting) { | ||
| 39 | return user.NotificationSetting.newFollow | ||
| 40 | } | ||
| 41 | |||
| 42 | getTargetUsers () { | ||
| 43 | if (!this.user) return [] | ||
| 44 | |||
| 45 | return [ this.user ] | ||
| 46 | } | ||
| 47 | |||
| 48 | createNotification (user: MUserWithNotificationSetting) { | ||
| 49 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 50 | type: UserNotificationType.NEW_FOLLOW, | ||
| 51 | userId: user.id, | ||
| 52 | actorFollowId: this.actorFollow.id | ||
| 53 | }) | ||
| 54 | notification.ActorFollow = this.actorFollow | ||
| 55 | |||
| 56 | return notification | ||
| 57 | } | ||
| 58 | |||
| 59 | createEmail (to: string) { | ||
| 60 | const following = this.actorFollow.ActorFollowing | ||
| 61 | const follower = this.actorFollow.ActorFollower | ||
| 62 | |||
| 63 | const followingName = (following.VideoChannel || following.Account).getDisplayName() | ||
| 64 | |||
| 65 | return { | ||
| 66 | template: 'follower-on-channel', | ||
| 67 | to, | ||
| 68 | subject: `New follower on your channel ${followingName}`, | ||
| 69 | locals: { | ||
| 70 | followerName: follower.Account.getDisplayName(), | ||
| 71 | followerUrl: follower.url, | ||
| 72 | followingName, | ||
| 73 | followingUrl: following.url, | ||
| 74 | followType: this.followType | ||
| 75 | } | ||
| 76 | } | ||
| 77 | } | ||
| 78 | |||
| 79 | private get actorFollow () { | ||
| 80 | return this.payload | ||
| 81 | } | ||
| 82 | } | ||
diff --git a/server/lib/notifier/shared/follow/index.ts b/server/lib/notifier/shared/follow/index.ts deleted file mode 100644 index 27f5289d9..000000000 --- a/server/lib/notifier/shared/follow/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './auto-follow-for-instance' | ||
| 2 | export * from './follow-for-instance' | ||
| 3 | export * from './follow-for-user' | ||
diff --git a/server/lib/notifier/shared/index.ts b/server/lib/notifier/shared/index.ts deleted file mode 100644 index cc3ce8c7c..000000000 --- a/server/lib/notifier/shared/index.ts +++ /dev/null | |||
| @@ -1,7 +0,0 @@ | |||
| 1 | export * from './abuse' | ||
| 2 | export * from './blacklist' | ||
| 3 | export * from './comment' | ||
| 4 | export * from './common' | ||
| 5 | export * from './follow' | ||
| 6 | export * from './instance' | ||
| 7 | export * from './video-publication' | ||
diff --git a/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts b/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts deleted file mode 100644 index 5044f2068..000000000 --- a/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts +++ /dev/null | |||
| @@ -1,49 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 6 | import { UserNotificationType, UserRight } from '@shared/models' | ||
| 7 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 8 | |||
| 9 | export class DirectRegistrationForModerators extends AbstractNotification <MUserDefault> { | ||
| 10 | private moderators: MUserDefault[] | ||
| 11 | |||
| 12 | async prepare () { | ||
| 13 | this.moderators = await UserModel.listWithRight(UserRight.MANAGE_USERS) | ||
| 14 | } | ||
| 15 | |||
| 16 | log () { | ||
| 17 | logger.info('Notifying %s moderators of new user registration of %s.', this.moderators.length, this.payload.username) | ||
| 18 | } | ||
| 19 | |||
| 20 | getSetting (user: MUserWithNotificationSetting) { | ||
| 21 | return user.NotificationSetting.newUserRegistration | ||
| 22 | } | ||
| 23 | |||
| 24 | getTargetUsers () { | ||
| 25 | return this.moderators | ||
| 26 | } | ||
| 27 | |||
| 28 | createNotification (user: MUserWithNotificationSetting) { | ||
| 29 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 30 | type: UserNotificationType.NEW_USER_REGISTRATION, | ||
| 31 | userId: user.id, | ||
| 32 | accountId: this.payload.Account.id | ||
| 33 | }) | ||
| 34 | notification.Account = this.payload.Account | ||
| 35 | |||
| 36 | return notification | ||
| 37 | } | ||
| 38 | |||
| 39 | createEmail (to: string) { | ||
| 40 | return { | ||
| 41 | template: 'user-registered', | ||
| 42 | to, | ||
| 43 | subject: `A new user registered on ${CONFIG.INSTANCE.NAME}: ${this.payload.username}`, | ||
| 44 | locals: { | ||
| 45 | user: this.payload | ||
| 46 | } | ||
| 47 | } | ||
| 48 | } | ||
| 49 | } | ||
diff --git a/server/lib/notifier/shared/instance/index.ts b/server/lib/notifier/shared/instance/index.ts deleted file mode 100644 index 8c75a8ee9..000000000 --- a/server/lib/notifier/shared/instance/index.ts +++ /dev/null | |||
| @@ -1,4 +0,0 @@ | |||
| 1 | export * from './new-peertube-version-for-admins' | ||
| 2 | export * from './new-plugin-version-for-admins' | ||
| 3 | export * from './direct-registration-for-moderators' | ||
| 4 | export * from './registration-request-for-moderators' | ||
diff --git a/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts b/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts deleted file mode 100644 index f5646c666..000000000 --- a/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts +++ /dev/null | |||
| @@ -1,54 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { UserModel } from '@server/models/user/user' | ||
| 3 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 4 | import { MApplication, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 5 | import { UserNotificationType, UserRight } from '@shared/models' | ||
| 6 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 7 | |||
| 8 | export type NewPeerTubeVersionForAdminsPayload = { | ||
| 9 | application: MApplication | ||
| 10 | latestVersion: string | ||
| 11 | } | ||
| 12 | |||
| 13 | export class NewPeerTubeVersionForAdmins extends AbstractNotification <NewPeerTubeVersionForAdminsPayload> { | ||
| 14 | private admins: MUserDefault[] | ||
| 15 | |||
| 16 | async prepare () { | ||
| 17 | // Use the debug right to know who is an administrator | ||
| 18 | this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) | ||
| 19 | } | ||
| 20 | |||
| 21 | log () { | ||
| 22 | logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) | ||
| 23 | } | ||
| 24 | |||
| 25 | getSetting (user: MUserWithNotificationSetting) { | ||
| 26 | return user.NotificationSetting.newPeerTubeVersion | ||
| 27 | } | ||
| 28 | |||
| 29 | getTargetUsers () { | ||
| 30 | return this.admins | ||
| 31 | } | ||
| 32 | |||
| 33 | createNotification (user: MUserWithNotificationSetting) { | ||
| 34 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 35 | type: UserNotificationType.NEW_PEERTUBE_VERSION, | ||
| 36 | userId: user.id, | ||
| 37 | applicationId: this.payload.application.id | ||
| 38 | }) | ||
| 39 | notification.Application = this.payload.application | ||
| 40 | |||
| 41 | return notification | ||
| 42 | } | ||
| 43 | |||
| 44 | createEmail (to: string) { | ||
| 45 | return { | ||
| 46 | to, | ||
| 47 | template: 'peertube-version-new', | ||
| 48 | subject: `A new PeerTube version is available: ${this.payload.latestVersion}`, | ||
| 49 | locals: { | ||
| 50 | latestVersion: this.payload.latestVersion | ||
| 51 | } | ||
| 52 | } | ||
| 53 | } | ||
| 54 | } | ||
diff --git a/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts b/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts deleted file mode 100644 index 547c6726c..000000000 --- a/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { MPlugin, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 6 | import { UserNotificationType, UserRight } from '@shared/models' | ||
| 7 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 8 | |||
| 9 | export class NewPluginVersionForAdmins extends AbstractNotification <MPlugin> { | ||
| 10 | private admins: MUserDefault[] | ||
| 11 | |||
| 12 | async prepare () { | ||
| 13 | // Use the debug right to know who is an administrator | ||
| 14 | this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) | ||
| 15 | } | ||
| 16 | |||
| 17 | log () { | ||
| 18 | logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) | ||
| 19 | } | ||
| 20 | |||
| 21 | getSetting (user: MUserWithNotificationSetting) { | ||
| 22 | return user.NotificationSetting.newPluginVersion | ||
| 23 | } | ||
| 24 | |||
| 25 | getTargetUsers () { | ||
| 26 | return this.admins | ||
| 27 | } | ||
| 28 | |||
| 29 | createNotification (user: MUserWithNotificationSetting) { | ||
| 30 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 31 | type: UserNotificationType.NEW_PLUGIN_VERSION, | ||
| 32 | userId: user.id, | ||
| 33 | pluginId: this.plugin.id | ||
| 34 | }) | ||
| 35 | notification.Plugin = this.plugin | ||
| 36 | |||
| 37 | return notification | ||
| 38 | } | ||
| 39 | |||
| 40 | createEmail (to: string) { | ||
| 41 | const pluginUrl = WEBSERVER.URL + '/admin/plugins/list-installed?pluginType=' + this.plugin.type | ||
| 42 | |||
| 43 | return { | ||
| 44 | to, | ||
| 45 | template: 'plugin-version-new', | ||
| 46 | subject: `A new plugin/theme version is available: ${this.plugin.name}@${this.plugin.latestVersion}`, | ||
| 47 | locals: { | ||
| 48 | pluginName: this.plugin.name, | ||
| 49 | latestVersion: this.plugin.latestVersion, | ||
| 50 | pluginUrl | ||
| 51 | } | ||
| 52 | } | ||
| 53 | } | ||
| 54 | |||
| 55 | private get plugin () { | ||
| 56 | return this.payload | ||
| 57 | } | ||
| 58 | } | ||
diff --git a/server/lib/notifier/shared/instance/registration-request-for-moderators.ts b/server/lib/notifier/shared/instance/registration-request-for-moderators.ts deleted file mode 100644 index 79920245a..000000000 --- a/server/lib/notifier/shared/instance/registration-request-for-moderators.ts +++ /dev/null | |||
| @@ -1,48 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { UserModel } from '@server/models/user/user' | ||
| 3 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 4 | import { MRegistration, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' | ||
| 5 | import { UserNotificationType, UserRight } from '@shared/models' | ||
| 6 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 7 | |||
| 8 | export class RegistrationRequestForModerators extends AbstractNotification <MRegistration> { | ||
| 9 | private moderators: MUserDefault[] | ||
| 10 | |||
| 11 | async prepare () { | ||
| 12 | this.moderators = await UserModel.listWithRight(UserRight.MANAGE_REGISTRATIONS) | ||
| 13 | } | ||
| 14 | |||
| 15 | log () { | ||
| 16 | logger.info('Notifying %s moderators of new user registration request of %s.', this.moderators.length, this.payload.username) | ||
| 17 | } | ||
| 18 | |||
| 19 | getSetting (user: MUserWithNotificationSetting) { | ||
| 20 | return user.NotificationSetting.newUserRegistration | ||
| 21 | } | ||
| 22 | |||
| 23 | getTargetUsers () { | ||
| 24 | return this.moderators | ||
| 25 | } | ||
| 26 | |||
| 27 | createNotification (user: MUserWithNotificationSetting) { | ||
| 28 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 29 | type: UserNotificationType.NEW_USER_REGISTRATION_REQUEST, | ||
| 30 | userId: user.id, | ||
| 31 | userRegistrationId: this.payload.id | ||
| 32 | }) | ||
| 33 | notification.UserRegistration = this.payload | ||
| 34 | |||
| 35 | return notification | ||
| 36 | } | ||
| 37 | |||
| 38 | createEmail (to: string) { | ||
| 39 | return { | ||
| 40 | template: 'user-registration-request', | ||
| 41 | to, | ||
| 42 | subject: `A new user wants to register: ${this.payload.username}`, | ||
| 43 | locals: { | ||
| 44 | registration: this.payload | ||
| 45 | } | ||
| 46 | } | ||
| 47 | } | ||
| 48 | } | ||
diff --git a/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts b/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts deleted file mode 100644 index a940cde69..000000000 --- a/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts +++ /dev/null | |||
| @@ -1,57 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' | ||
| 6 | import { UserNotificationType } from '@shared/models' | ||
| 7 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 8 | |||
| 9 | export abstract class AbstractOwnedVideoPublication extends AbstractNotification <MVideoFullLight> { | ||
| 10 | protected user: MUserDefault | ||
| 11 | |||
| 12 | async prepare () { | ||
| 13 | this.user = await UserModel.loadByVideoId(this.payload.id) | ||
| 14 | } | ||
| 15 | |||
| 16 | log () { | ||
| 17 | logger.info('Notifying user %s of the publication of its video %s.', this.user.username, this.payload.url) | ||
| 18 | } | ||
| 19 | |||
| 20 | getSetting (user: MUserWithNotificationSetting) { | ||
| 21 | return user.NotificationSetting.myVideoPublished | ||
| 22 | } | ||
| 23 | |||
| 24 | getTargetUsers () { | ||
| 25 | if (!this.user) return [] | ||
| 26 | |||
| 27 | return [ this.user ] | ||
| 28 | } | ||
| 29 | |||
| 30 | createNotification (user: MUserWithNotificationSetting) { | ||
| 31 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 32 | type: UserNotificationType.MY_VIDEO_PUBLISHED, | ||
| 33 | userId: user.id, | ||
| 34 | videoId: this.payload.id | ||
| 35 | }) | ||
| 36 | notification.Video = this.payload | ||
| 37 | |||
| 38 | return notification | ||
| 39 | } | ||
| 40 | |||
| 41 | createEmail (to: string) { | ||
| 42 | const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() | ||
| 43 | |||
| 44 | return { | ||
| 45 | to, | ||
| 46 | subject: `Your video ${this.payload.name} has been published`, | ||
| 47 | text: `Your video "${this.payload.name}" has been published.`, | ||
| 48 | locals: { | ||
| 49 | title: 'Your video is live', | ||
| 50 | action: { | ||
| 51 | text: 'View video', | ||
| 52 | url: videoUrl | ||
| 53 | } | ||
| 54 | } | ||
| 55 | } | ||
| 56 | } | ||
| 57 | } | ||
diff --git a/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts b/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts deleted file mode 100644 index 3bd64692f..000000000 --- a/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts +++ /dev/null | |||
| @@ -1,97 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { MUserDefault, MUserWithNotificationSetting, MVideoImportVideo, UserNotificationModelForApi } from '@server/types/models' | ||
| 6 | import { UserNotificationType } from '@shared/models' | ||
| 7 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 8 | |||
| 9 | export type ImportFinishedForOwnerPayload = { | ||
| 10 | videoImport: MVideoImportVideo | ||
| 11 | success: boolean | ||
| 12 | } | ||
| 13 | |||
| 14 | export class ImportFinishedForOwner extends AbstractNotification <ImportFinishedForOwnerPayload> { | ||
| 15 | private user: MUserDefault | ||
| 16 | |||
| 17 | async prepare () { | ||
| 18 | this.user = await UserModel.loadByVideoImportId(this.videoImport.id) | ||
| 19 | } | ||
| 20 | |||
| 21 | log () { | ||
| 22 | logger.info('Notifying user %s its video import %s is finished.', this.user.username, this.videoImport.getTargetIdentifier()) | ||
| 23 | } | ||
| 24 | |||
| 25 | getSetting (user: MUserWithNotificationSetting) { | ||
| 26 | return user.NotificationSetting.myVideoImportFinished | ||
| 27 | } | ||
| 28 | |||
| 29 | getTargetUsers () { | ||
| 30 | if (!this.user) return [] | ||
| 31 | |||
| 32 | return [ this.user ] | ||
| 33 | } | ||
| 34 | |||
| 35 | createNotification (user: MUserWithNotificationSetting) { | ||
| 36 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 37 | type: this.payload.success | ||
| 38 | ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS | ||
| 39 | : UserNotificationType.MY_VIDEO_IMPORT_ERROR, | ||
| 40 | |||
| 41 | userId: user.id, | ||
| 42 | videoImportId: this.videoImport.id | ||
| 43 | }) | ||
| 44 | notification.VideoImport = this.videoImport | ||
| 45 | |||
| 46 | return notification | ||
| 47 | } | ||
| 48 | |||
| 49 | createEmail (to: string) { | ||
| 50 | if (this.payload.success) return this.createSuccessEmail(to) | ||
| 51 | |||
| 52 | return this.createFailEmail(to) | ||
| 53 | } | ||
| 54 | |||
| 55 | private createSuccessEmail (to: string) { | ||
| 56 | const videoUrl = WEBSERVER.URL + this.videoImport.Video.getWatchStaticPath() | ||
| 57 | |||
| 58 | return { | ||
| 59 | to, | ||
| 60 | subject: `Your video import ${this.videoImport.getTargetIdentifier()} is complete`, | ||
| 61 | text: `Your video "${this.videoImport.getTargetIdentifier()}" just finished importing.`, | ||
| 62 | locals: { | ||
| 63 | title: 'Import complete', | ||
| 64 | action: { | ||
| 65 | text: 'View video', | ||
| 66 | url: videoUrl | ||
| 67 | } | ||
| 68 | } | ||
| 69 | } | ||
| 70 | } | ||
| 71 | |||
| 72 | private createFailEmail (to: string) { | ||
| 73 | const importUrl = WEBSERVER.URL + '/my-library/video-imports' | ||
| 74 | |||
| 75 | const text = | ||
| 76 | `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error.` + | ||
| 77 | '\n\n' + | ||
| 78 | `See your videos import dashboard for more information: <a href="${importUrl}">${importUrl}</a>.` | ||
| 79 | |||
| 80 | return { | ||
| 81 | to, | ||
| 82 | subject: `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error`, | ||
| 83 | text, | ||
| 84 | locals: { | ||
| 85 | title: 'Import failed', | ||
| 86 | action: { | ||
| 87 | text: 'Review imports', | ||
| 88 | url: importUrl | ||
| 89 | } | ||
| 90 | } | ||
| 91 | } | ||
| 92 | } | ||
| 93 | |||
| 94 | private get videoImport () { | ||
| 95 | return this.payload.videoImport | ||
| 96 | } | ||
| 97 | } | ||
diff --git a/server/lib/notifier/shared/video-publication/index.ts b/server/lib/notifier/shared/video-publication/index.ts deleted file mode 100644 index 5e92cb011..000000000 --- a/server/lib/notifier/shared/video-publication/index.ts +++ /dev/null | |||
| @@ -1,6 +0,0 @@ | |||
| 1 | export * from './new-video-for-subscribers' | ||
| 2 | export * from './import-finished-for-owner' | ||
| 3 | export * from './owned-publication-after-auto-unblacklist' | ||
| 4 | export * from './owned-publication-after-schedule-update' | ||
| 5 | export * from './owned-publication-after-transcoding' | ||
| 6 | export * from './studio-edition-finished-for-owner' | ||
diff --git a/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts b/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts deleted file mode 100644 index df7a5561d..000000000 --- a/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts +++ /dev/null | |||
| @@ -1,61 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { MUserWithNotificationSetting, MVideoAccountLight, UserNotificationModelForApi } from '@server/types/models' | ||
| 6 | import { UserNotificationType, VideoPrivacy, VideoState } from '@shared/models' | ||
| 7 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 8 | |||
| 9 | export class NewVideoForSubscribers extends AbstractNotification <MVideoAccountLight> { | ||
| 10 | private users: MUserWithNotificationSetting[] | ||
| 11 | |||
| 12 | async prepare () { | ||
| 13 | // List all followers that are users | ||
| 14 | this.users = await UserModel.listUserSubscribersOf(this.payload.VideoChannel.actorId) | ||
| 15 | } | ||
| 16 | |||
| 17 | log () { | ||
| 18 | logger.info('Notifying %d users of new video %s.', this.users.length, this.payload.url) | ||
| 19 | } | ||
| 20 | |||
| 21 | isDisabled () { | ||
| 22 | return this.payload.privacy !== VideoPrivacy.PUBLIC || this.payload.state !== VideoState.PUBLISHED || this.payload.isBlacklisted() | ||
| 23 | } | ||
| 24 | |||
| 25 | getSetting (user: MUserWithNotificationSetting) { | ||
| 26 | return user.NotificationSetting.newVideoFromSubscription | ||
| 27 | } | ||
| 28 | |||
| 29 | getTargetUsers () { | ||
| 30 | return this.users | ||
| 31 | } | ||
| 32 | |||
| 33 | createNotification (user: MUserWithNotificationSetting) { | ||
| 34 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 35 | type: UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION, | ||
| 36 | userId: user.id, | ||
| 37 | videoId: this.payload.id | ||
| 38 | }) | ||
| 39 | notification.Video = this.payload | ||
| 40 | |||
| 41 | return notification | ||
| 42 | } | ||
| 43 | |||
| 44 | createEmail (to: string) { | ||
| 45 | const channelName = this.payload.VideoChannel.getDisplayName() | ||
| 46 | const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() | ||
| 47 | |||
| 48 | return { | ||
| 49 | to, | ||
| 50 | subject: channelName + ' just published a new video', | ||
| 51 | text: `Your subscription ${channelName} just published a new video: "${this.payload.name}".`, | ||
| 52 | locals: { | ||
| 53 | title: 'New content ', | ||
| 54 | action: { | ||
| 55 | text: 'View video', | ||
| 56 | url: videoUrl | ||
| 57 | } | ||
| 58 | } | ||
| 59 | } | ||
| 60 | } | ||
| 61 | } | ||
diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts deleted file mode 100644 index 27d89a5c7..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts +++ /dev/null | |||
| @@ -1,11 +0,0 @@ | |||
| 1 | |||
| 2 | import { VideoState } from '@shared/models' | ||
| 3 | import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' | ||
| 4 | |||
| 5 | export class OwnedPublicationAfterAutoUnblacklist extends AbstractOwnedVideoPublication { | ||
| 6 | |||
| 7 | isDisabled () { | ||
| 8 | // Don't notify if video is still waiting for transcoding or scheduled update | ||
| 9 | return !!this.payload.ScheduleVideoUpdate || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) | ||
| 10 | } | ||
| 11 | } | ||
diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts deleted file mode 100644 index 2e253b358..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts +++ /dev/null | |||
| @@ -1,10 +0,0 @@ | |||
| 1 | import { VideoState } from '@shared/models' | ||
| 2 | import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' | ||
| 3 | |||
| 4 | export class OwnedPublicationAfterScheduleUpdate extends AbstractOwnedVideoPublication { | ||
| 5 | |||
| 6 | isDisabled () { | ||
| 7 | // Don't notify if video is still blacklisted or waiting for transcoding | ||
| 8 | return !!this.payload.VideoBlacklist || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) | ||
| 9 | } | ||
| 10 | } | ||
diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts deleted file mode 100644 index 4fab1090f..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' | ||
| 2 | |||
| 3 | export class OwnedPublicationAfterTranscoding extends AbstractOwnedVideoPublication { | ||
| 4 | |||
| 5 | isDisabled () { | ||
| 6 | // Don't notify if didn't wait for transcoding or video is still blacklisted/waiting for scheduled update | ||
| 7 | return !this.payload.waitTranscoding || !!this.payload.VideoBlacklist || !!this.payload.ScheduleVideoUpdate | ||
| 8 | } | ||
| 9 | } | ||
diff --git a/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts b/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts deleted file mode 100644 index f36399f05..000000000 --- a/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts +++ /dev/null | |||
| @@ -1,57 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { UserModel } from '@server/models/user/user' | ||
| 4 | import { UserNotificationModel } from '@server/models/user/user-notification' | ||
| 5 | import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' | ||
| 6 | import { UserNotificationType } from '@shared/models' | ||
| 7 | import { AbstractNotification } from '../common/abstract-notification' | ||
| 8 | |||
| 9 | export class StudioEditionFinishedForOwner extends AbstractNotification <MVideoFullLight> { | ||
| 10 | private user: MUserDefault | ||
| 11 | |||
| 12 | async prepare () { | ||
| 13 | this.user = await UserModel.loadByVideoId(this.payload.id) | ||
| 14 | } | ||
| 15 | |||
| 16 | log () { | ||
| 17 | logger.info('Notifying user %s its video studio edition %s is finished.', this.user.username, this.payload.url) | ||
| 18 | } | ||
| 19 | |||
| 20 | getSetting (user: MUserWithNotificationSetting) { | ||
| 21 | return user.NotificationSetting.myVideoStudioEditionFinished | ||
| 22 | } | ||
| 23 | |||
| 24 | getTargetUsers () { | ||
| 25 | if (!this.user) return [] | ||
| 26 | |||
| 27 | return [ this.user ] | ||
| 28 | } | ||
| 29 | |||
| 30 | createNotification (user: MUserWithNotificationSetting) { | ||
| 31 | const notification = UserNotificationModel.build<UserNotificationModelForApi>({ | ||
| 32 | type: UserNotificationType.MY_VIDEO_STUDIO_EDITION_FINISHED, | ||
| 33 | userId: user.id, | ||
| 34 | videoId: this.payload.id | ||
| 35 | }) | ||
| 36 | notification.Video = this.payload | ||
| 37 | |||
| 38 | return notification | ||
| 39 | } | ||
| 40 | |||
| 41 | createEmail (to: string) { | ||
| 42 | const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() | ||
| 43 | |||
| 44 | return { | ||
| 45 | to, | ||
| 46 | subject: `Edition of your video ${this.payload.name} has finished`, | ||
| 47 | text: `Edition of your video ${this.payload.name} has finished.`, | ||
| 48 | locals: { | ||
| 49 | title: 'Video edition has finished', | ||
| 50 | action: { | ||
| 51 | text: 'View video', | ||
| 52 | url: videoUrl | ||
| 53 | } | ||
| 54 | } | ||
| 55 | } | ||
| 56 | } | ||
| 57 | } | ||
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts deleted file mode 100644 index 3ad6cab63..000000000 --- a/server/lib/object-storage/index.ts +++ /dev/null | |||
| @@ -1,5 +0,0 @@ | |||
| 1 | export * from './keys' | ||
| 2 | export * from './proxy' | ||
| 3 | export * from './pre-signed-urls' | ||
| 4 | export * from './urls' | ||
| 5 | export * from './videos' | ||
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts deleted file mode 100644 index 6d2098298..000000000 --- a/server/lib/object-storage/keys.ts +++ /dev/null | |||
| @@ -1,20 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import { MStreamingPlaylistVideo } from '@server/types/models' | ||
| 3 | |||
| 4 | function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { | ||
| 5 | return join(generateHLSObjectBaseStorageKey(playlist), filename) | ||
| 6 | } | ||
| 7 | |||
| 8 | function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { | ||
| 9 | return join(playlist.getStringType(), playlist.Video.uuid) | ||
| 10 | } | ||
| 11 | |||
| 12 | function generateWebVideoObjectStorageKey (filename: string) { | ||
| 13 | return filename | ||
| 14 | } | ||
| 15 | |||
| 16 | export { | ||
| 17 | generateHLSObjectStorageKey, | ||
| 18 | generateHLSObjectBaseStorageKey, | ||
| 19 | generateWebVideoObjectStorageKey | ||
| 20 | } | ||
diff --git a/server/lib/object-storage/pre-signed-urls.ts b/server/lib/object-storage/pre-signed-urls.ts deleted file mode 100644 index caf149bb8..000000000 --- a/server/lib/object-storage/pre-signed-urls.ts +++ /dev/null | |||
| @@ -1,46 +0,0 @@ | |||
| 1 | import { GetObjectCommand } from '@aws-sdk/client-s3' | ||
| 2 | import { getSignedUrl } from '@aws-sdk/s3-request-presigner' | ||
| 3 | import { CONFIG } from '@server/initializers/config' | ||
| 4 | import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' | ||
| 5 | import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' | ||
| 6 | import { buildKey, getClient } from './shared' | ||
| 7 | import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls' | ||
| 8 | |||
| 9 | export async function generateWebVideoPresignedUrl (options: { | ||
| 10 | file: MVideoFile | ||
| 11 | downloadFilename: string | ||
| 12 | }) { | ||
| 13 | const { file, downloadFilename } = options | ||
| 14 | |||
| 15 | const key = generateWebVideoObjectStorageKey(file.filename) | ||
| 16 | |||
| 17 | const command = new GetObjectCommand({ | ||
| 18 | Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, | ||
| 19 | Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS), | ||
| 20 | ResponseContentDisposition: `attachment; filename=${downloadFilename}` | ||
| 21 | }) | ||
| 22 | |||
| 23 | const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) | ||
| 24 | |||
| 25 | return getWebVideoPublicFileUrl(url) | ||
| 26 | } | ||
| 27 | |||
| 28 | export async function generateHLSFilePresignedUrl (options: { | ||
| 29 | streamingPlaylist: MStreamingPlaylistVideo | ||
| 30 | file: MVideoFile | ||
| 31 | downloadFilename: string | ||
| 32 | }) { | ||
| 33 | const { streamingPlaylist, file, downloadFilename } = options | ||
| 34 | |||
| 35 | const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename) | ||
| 36 | |||
| 37 | const command = new GetObjectCommand({ | ||
| 38 | Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, | ||
| 39 | Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), | ||
| 40 | ResponseContentDisposition: `attachment; filename=${downloadFilename}` | ||
| 41 | }) | ||
| 42 | |||
| 43 | const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) | ||
| 44 | |||
| 45 | return getHLSPublicFileUrl(url) | ||
| 46 | } | ||
diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts deleted file mode 100644 index c09a0d1b0..000000000 --- a/server/lib/object-storage/proxy.ts +++ /dev/null | |||
| @@ -1,97 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { PassThrough, pipeline } from 'stream' | ||
| 3 | import { GetObjectCommandOutput } from '@aws-sdk/client-s3' | ||
| 4 | import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist' | ||
| 5 | import { logger } from '@server/helpers/logger' | ||
| 6 | import { StreamReplacer } from '@server/helpers/stream-replacer' | ||
| 7 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | ||
| 8 | import { HttpStatusCode } from '@shared/models' | ||
| 9 | import { injectQueryToPlaylistUrls } from '../hls' | ||
| 10 | import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos' | ||
| 11 | |||
| 12 | export async function proxifyWebVideoFile (options: { | ||
| 13 | req: express.Request | ||
| 14 | res: express.Response | ||
| 15 | filename: string | ||
| 16 | }) { | ||
| 17 | const { req, res, filename } = options | ||
| 18 | |||
| 19 | logger.debug('Proxifying Web Video file %s from object storage.', filename) | ||
| 20 | |||
| 21 | try { | ||
| 22 | const { response: s3Response, stream } = await getWebVideoFileReadStream({ | ||
| 23 | filename, | ||
| 24 | rangeHeader: req.header('range') | ||
| 25 | }) | ||
| 26 | |||
| 27 | setS3Headers(res, s3Response) | ||
| 28 | |||
| 29 | return stream.pipe(res) | ||
| 30 | } catch (err) { | ||
| 31 | return handleObjectStorageFailure(res, err) | ||
| 32 | } | ||
| 33 | } | ||
| 34 | |||
| 35 | export async function proxifyHLS (options: { | ||
| 36 | req: express.Request | ||
| 37 | res: express.Response | ||
| 38 | playlist: MStreamingPlaylist | ||
| 39 | video: MVideo | ||
| 40 | filename: string | ||
| 41 | reinjectVideoFileToken: boolean | ||
| 42 | }) { | ||
| 43 | const { req, res, playlist, video, filename, reinjectVideoFileToken } = options | ||
| 44 | |||
| 45 | logger.debug('Proxifying HLS file %s from object storage.', filename) | ||
| 46 | |||
| 47 | try { | ||
| 48 | const { response: s3Response, stream } = await getHLSFileReadStream({ | ||
| 49 | playlist: playlist.withVideo(video), | ||
| 50 | filename, | ||
| 51 | rangeHeader: req.header('range') | ||
| 52 | }) | ||
| 53 | |||
| 54 | setS3Headers(res, s3Response) | ||
| 55 | |||
| 56 | const streamReplacer = reinjectVideoFileToken | ||
| 57 | ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) | ||
| 58 | : new PassThrough() | ||
| 59 | |||
| 60 | return pipeline( | ||
| 61 | stream, | ||
| 62 | streamReplacer, | ||
| 63 | res, | ||
| 64 | err => { | ||
| 65 | if (!err) return | ||
| 66 | |||
| 67 | handleObjectStorageFailure(res, err) | ||
| 68 | } | ||
| 69 | ) | ||
| 70 | } catch (err) { | ||
| 71 | return handleObjectStorageFailure(res, err) | ||
| 72 | } | ||
| 73 | } | ||
| 74 | |||
| 75 | // --------------------------------------------------------------------------- | ||
| 76 | // Private | ||
| 77 | // --------------------------------------------------------------------------- | ||
| 78 | |||
| 79 | function handleObjectStorageFailure (res: express.Response, err: Error) { | ||
| 80 | if (err.name === 'NoSuchKey') { | ||
| 81 | logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) | ||
| 82 | return res.sendStatus(HttpStatusCode.NOT_FOUND_404) | ||
| 83 | } | ||
| 84 | |||
| 85 | return res.fail({ | ||
| 86 | status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, | ||
| 87 | message: err.message, | ||
| 88 | type: err.name | ||
| 89 | }) | ||
| 90 | } | ||
| 91 | |||
| 92 | function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { | ||
| 93 | if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { | ||
| 94 | res.setHeader('Content-Range', s3Response.ContentRange) | ||
| 95 | res.status(HttpStatusCode.PARTIAL_CONTENT_206) | ||
| 96 | } | ||
| 97 | } | ||
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts deleted file mode 100644 index d5cb074df..000000000 --- a/server/lib/object-storage/shared/client.ts +++ /dev/null | |||
| @@ -1,71 +0,0 @@ | |||
| 1 | import { S3Client } from '@aws-sdk/client-s3' | ||
| 2 | import { NodeHttpHandler } from '@aws-sdk/node-http-handler' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { isProxyEnabled } from '@server/helpers/proxy' | ||
| 5 | import { getAgent } from '@server/helpers/requests' | ||
| 6 | import { CONFIG } from '@server/initializers/config' | ||
| 7 | import { lTags } from './logger' | ||
| 8 | |||
| 9 | function getProxyRequestHandler () { | ||
| 10 | if (!isProxyEnabled()) return null | ||
| 11 | |||
| 12 | const { agent } = getAgent() | ||
| 13 | |||
| 14 | return new NodeHttpHandler({ | ||
| 15 | httpAgent: agent.http, | ||
| 16 | httpsAgent: agent.https | ||
| 17 | }) | ||
| 18 | } | ||
| 19 | |||
| 20 | let endpointParsed: URL | ||
| 21 | function getEndpointParsed () { | ||
| 22 | if (endpointParsed) return endpointParsed | ||
| 23 | |||
| 24 | endpointParsed = new URL(getEndpoint()) | ||
| 25 | |||
| 26 | return endpointParsed | ||
| 27 | } | ||
| 28 | |||
| 29 | let s3Client: S3Client | ||
| 30 | function getClient () { | ||
| 31 | if (s3Client) return s3Client | ||
| 32 | |||
| 33 | const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE | ||
| 34 | |||
| 35 | s3Client = new S3Client({ | ||
| 36 | endpoint: getEndpoint(), | ||
| 37 | region: OBJECT_STORAGE.REGION, | ||
| 38 | credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID | ||
| 39 | ? { | ||
| 40 | accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, | ||
| 41 | secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY | ||
| 42 | } | ||
| 43 | : undefined, | ||
| 44 | requestHandler: getProxyRequestHandler() | ||
| 45 | }) | ||
| 46 | |||
| 47 | logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) | ||
| 48 | |||
| 49 | return s3Client | ||
| 50 | } | ||
| 51 | |||
| 52 | // --------------------------------------------------------------------------- | ||
| 53 | |||
| 54 | export { | ||
| 55 | getEndpointParsed, | ||
| 56 | getClient | ||
| 57 | } | ||
| 58 | |||
| 59 | // --------------------------------------------------------------------------- | ||
| 60 | |||
| 61 | let endpoint: string | ||
| 62 | function getEndpoint () { | ||
| 63 | if (endpoint) return endpoint | ||
| 64 | |||
| 65 | const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT | ||
| 66 | endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') | ||
| 67 | ? CONFIG.OBJECT_STORAGE.ENDPOINT | ||
| 68 | : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT | ||
| 69 | |||
| 70 | return endpoint | ||
| 71 | } | ||
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts deleted file mode 100644 index 11e10aa9f..000000000 --- a/server/lib/object-storage/shared/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './client' | ||
| 2 | export * from './logger' | ||
| 3 | export * from './object-storage-helpers' | ||
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts deleted file mode 100644 index 8ab7cbd71..000000000 --- a/server/lib/object-storage/shared/logger.ts +++ /dev/null | |||
| @@ -1,7 +0,0 @@ | |||
| 1 | import { loggerTagsFactory } from '@server/helpers/logger' | ||
| 2 | |||
| 3 | const lTags = loggerTagsFactory('object-storage') | ||
| 4 | |||
| 5 | export { | ||
| 6 | lTags | ||
| 7 | } | ||
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts deleted file mode 100644 index 0d8878bd2..000000000 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ /dev/null | |||
| @@ -1,328 +0,0 @@ | |||
| 1 | import { map } from 'bluebird' | ||
| 2 | import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' | ||
| 3 | import { dirname } from 'path' | ||
| 4 | import { Readable } from 'stream' | ||
| 5 | import { | ||
| 6 | _Object, | ||
| 7 | CompleteMultipartUploadCommandOutput, | ||
| 8 | DeleteObjectCommand, | ||
| 9 | GetObjectCommand, | ||
| 10 | ListObjectsV2Command, | ||
| 11 | PutObjectAclCommand, | ||
| 12 | PutObjectCommandInput, | ||
| 13 | S3Client | ||
| 14 | } from '@aws-sdk/client-s3' | ||
| 15 | import { Upload } from '@aws-sdk/lib-storage' | ||
| 16 | import { pipelinePromise } from '@server/helpers/core-utils' | ||
| 17 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
| 18 | import { logger } from '@server/helpers/logger' | ||
| 19 | import { CONFIG } from '@server/initializers/config' | ||
| 20 | import { getInternalUrl } from '../urls' | ||
| 21 | import { getClient } from './client' | ||
| 22 | import { lTags } from './logger' | ||
| 23 | |||
| 24 | type BucketInfo = { | ||
| 25 | BUCKET_NAME: string | ||
| 26 | PREFIX?: string | ||
| 27 | } | ||
| 28 | |||
| 29 | async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { | ||
| 30 | const s3Client = getClient() | ||
| 31 | |||
| 32 | const commandPrefix = bucketInfo.PREFIX + prefix | ||
| 33 | const listCommand = new ListObjectsV2Command({ | ||
| 34 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 35 | Prefix: commandPrefix | ||
| 36 | }) | ||
| 37 | |||
| 38 | const listedObjects = await s3Client.send(listCommand) | ||
| 39 | |||
| 40 | if (isArray(listedObjects.Contents) !== true) return [] | ||
| 41 | |||
| 42 | return listedObjects.Contents.map(c => c.Key) | ||
| 43 | } | ||
| 44 | |||
| 45 | // --------------------------------------------------------------------------- | ||
| 46 | |||
| 47 | async function storeObject (options: { | ||
| 48 | inputPath: string | ||
| 49 | objectStorageKey: string | ||
| 50 | bucketInfo: BucketInfo | ||
| 51 | isPrivate: boolean | ||
| 52 | }): Promise<string> { | ||
| 53 | const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options | ||
| 54 | |||
| 55 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) | ||
| 56 | |||
| 57 | const fileStream = createReadStream(inputPath) | ||
| 58 | |||
| 59 | return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) | ||
| 60 | } | ||
| 61 | |||
| 62 | async function storeContent (options: { | ||
| 63 | content: string | ||
| 64 | inputPath: string | ||
| 65 | objectStorageKey: string | ||
| 66 | bucketInfo: BucketInfo | ||
| 67 | isPrivate: boolean | ||
| 68 | }): Promise<string> { | ||
| 69 | const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options | ||
| 70 | |||
| 71 | logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) | ||
| 72 | |||
| 73 | return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) | ||
| 74 | } | ||
| 75 | |||
| 76 | // --------------------------------------------------------------------------- | ||
| 77 | |||
| 78 | async function updateObjectACL (options: { | ||
| 79 | objectStorageKey: string | ||
| 80 | bucketInfo: BucketInfo | ||
| 81 | isPrivate: boolean | ||
| 82 | }) { | ||
| 83 | const { objectStorageKey, bucketInfo, isPrivate } = options | ||
| 84 | |||
| 85 | const acl = getACL(isPrivate) | ||
| 86 | if (!acl) return | ||
| 87 | |||
| 88 | const key = buildKey(objectStorageKey, bucketInfo) | ||
| 89 | |||
| 90 | logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) | ||
| 91 | |||
| 92 | const command = new PutObjectAclCommand({ | ||
| 93 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 94 | Key: key, | ||
| 95 | ACL: acl | ||
| 96 | }) | ||
| 97 | |||
| 98 | await getClient().send(command) | ||
| 99 | } | ||
| 100 | |||
| 101 | function updatePrefixACL (options: { | ||
| 102 | prefix: string | ||
| 103 | bucketInfo: BucketInfo | ||
| 104 | isPrivate: boolean | ||
| 105 | }) { | ||
| 106 | const { prefix, bucketInfo, isPrivate } = options | ||
| 107 | |||
| 108 | const acl = getACL(isPrivate) | ||
| 109 | if (!acl) return | ||
| 110 | |||
| 111 | logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
| 112 | |||
| 113 | return applyOnPrefix({ | ||
| 114 | prefix, | ||
| 115 | bucketInfo, | ||
| 116 | commandBuilder: obj => { | ||
| 117 | logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
| 118 | |||
| 119 | return new PutObjectAclCommand({ | ||
| 120 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 121 | Key: obj.Key, | ||
| 122 | ACL: acl | ||
| 123 | }) | ||
| 124 | } | ||
| 125 | }) | ||
| 126 | } | ||
| 127 | |||
| 128 | // --------------------------------------------------------------------------- | ||
| 129 | |||
| 130 | function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { | ||
| 131 | const key = buildKey(objectStorageKey, bucketInfo) | ||
| 132 | |||
| 133 | return removeObjectByFullKey(key, bucketInfo) | ||
| 134 | } | ||
| 135 | |||
| 136 | function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { | ||
| 137 | logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) | ||
| 138 | |||
| 139 | const command = new DeleteObjectCommand({ | ||
| 140 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 141 | Key: fullKey | ||
| 142 | }) | ||
| 143 | |||
| 144 | return getClient().send(command) | ||
| 145 | } | ||
| 146 | |||
| 147 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | ||
| 148 | logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
| 149 | |||
| 150 | return applyOnPrefix({ | ||
| 151 | prefix, | ||
| 152 | bucketInfo, | ||
| 153 | commandBuilder: obj => { | ||
| 154 | logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
| 155 | |||
| 156 | return new DeleteObjectCommand({ | ||
| 157 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 158 | Key: obj.Key | ||
| 159 | }) | ||
| 160 | } | ||
| 161 | }) | ||
| 162 | } | ||
| 163 | |||
| 164 | // --------------------------------------------------------------------------- | ||
| 165 | |||
| 166 | async function makeAvailable (options: { | ||
| 167 | key: string | ||
| 168 | destination: string | ||
| 169 | bucketInfo: BucketInfo | ||
| 170 | }) { | ||
| 171 | const { key, destination, bucketInfo } = options | ||
| 172 | |||
| 173 | await ensureDir(dirname(options.destination)) | ||
| 174 | |||
| 175 | const command = new GetObjectCommand({ | ||
| 176 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 177 | Key: buildKey(key, bucketInfo) | ||
| 178 | }) | ||
| 179 | const response = await getClient().send(command) | ||
| 180 | |||
| 181 | const file = createWriteStream(destination) | ||
| 182 | await pipelinePromise(response.Body as Readable, file) | ||
| 183 | |||
| 184 | file.close() | ||
| 185 | } | ||
| 186 | |||
| 187 | function buildKey (key: string, bucketInfo: BucketInfo) { | ||
| 188 | return bucketInfo.PREFIX + key | ||
| 189 | } | ||
| 190 | |||
| 191 | // --------------------------------------------------------------------------- | ||
| 192 | |||
| 193 | async function createObjectReadStream (options: { | ||
| 194 | key: string | ||
| 195 | bucketInfo: BucketInfo | ||
| 196 | rangeHeader: string | ||
| 197 | }) { | ||
| 198 | const { key, bucketInfo, rangeHeader } = options | ||
| 199 | |||
| 200 | const command = new GetObjectCommand({ | ||
| 201 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 202 | Key: buildKey(key, bucketInfo), | ||
| 203 | Range: rangeHeader | ||
| 204 | }) | ||
| 205 | |||
| 206 | const response = await getClient().send(command) | ||
| 207 | |||
| 208 | return { | ||
| 209 | response, | ||
| 210 | stream: response.Body as Readable | ||
| 211 | } | ||
| 212 | } | ||
| 213 | |||
| 214 | // --------------------------------------------------------------------------- | ||
| 215 | |||
| 216 | export { | ||
| 217 | BucketInfo, | ||
| 218 | buildKey, | ||
| 219 | |||
| 220 | storeObject, | ||
| 221 | storeContent, | ||
| 222 | |||
| 223 | removeObject, | ||
| 224 | removeObjectByFullKey, | ||
| 225 | removePrefix, | ||
| 226 | |||
| 227 | makeAvailable, | ||
| 228 | |||
| 229 | updateObjectACL, | ||
| 230 | updatePrefixACL, | ||
| 231 | |||
| 232 | listKeysOfPrefix, | ||
| 233 | createObjectReadStream | ||
| 234 | } | ||
| 235 | |||
| 236 | // --------------------------------------------------------------------------- | ||
| 237 | |||
| 238 | async function uploadToStorage (options: { | ||
| 239 | content: ReadStream | string | ||
| 240 | objectStorageKey: string | ||
| 241 | bucketInfo: BucketInfo | ||
| 242 | isPrivate: boolean | ||
| 243 | }) { | ||
| 244 | const { content, objectStorageKey, bucketInfo, isPrivate } = options | ||
| 245 | |||
| 246 | const input: PutObjectCommandInput = { | ||
| 247 | Body: content, | ||
| 248 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 249 | Key: buildKey(objectStorageKey, bucketInfo) | ||
| 250 | } | ||
| 251 | |||
| 252 | const acl = getACL(isPrivate) | ||
| 253 | if (acl) input.ACL = acl | ||
| 254 | |||
| 255 | const parallelUploads3 = new Upload({ | ||
| 256 | client: getClient(), | ||
| 257 | queueSize: 4, | ||
| 258 | partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART, | ||
| 259 | |||
| 260 | // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts | ||
| 261 | // More detailed explanation: | ||
| 262 | // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274 | ||
| 263 | // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928 | ||
| 264 | leavePartsOnError: true, | ||
| 265 | params: input | ||
| 266 | }) | ||
| 267 | |||
| 268 | const response = (await parallelUploads3.done()) as CompleteMultipartUploadCommandOutput | ||
| 269 | // Check is needed even if the HTTP status code is 200 OK | ||
| 270 | // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html | ||
| 271 | if (!response.Bucket) { | ||
| 272 | const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}` | ||
| 273 | logger.error(message, { response, ...lTags() }) | ||
| 274 | throw new Error(message) | ||
| 275 | } | ||
| 276 | |||
| 277 | logger.debug( | ||
| 278 | 'Completed %s%s in bucket %s', | ||
| 279 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata } | ||
| 280 | ) | ||
| 281 | |||
| 282 | return getInternalUrl(bucketInfo, objectStorageKey) | ||
| 283 | } | ||
| 284 | |||
| 285 | async function applyOnPrefix (options: { | ||
| 286 | prefix: string | ||
| 287 | bucketInfo: BucketInfo | ||
| 288 | commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0] | ||
| 289 | |||
| 290 | continuationToken?: string | ||
| 291 | }) { | ||
| 292 | const { prefix, bucketInfo, commandBuilder, continuationToken } = options | ||
| 293 | |||
| 294 | const s3Client = getClient() | ||
| 295 | |||
| 296 | const commandPrefix = buildKey(prefix, bucketInfo) | ||
| 297 | const listCommand = new ListObjectsV2Command({ | ||
| 298 | Bucket: bucketInfo.BUCKET_NAME, | ||
| 299 | Prefix: commandPrefix, | ||
| 300 | ContinuationToken: continuationToken | ||
| 301 | }) | ||
| 302 | |||
| 303 | const listedObjects = await s3Client.send(listCommand) | ||
| 304 | |||
| 305 | if (isArray(listedObjects.Contents) !== true) { | ||
| 306 | const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | ||
| 307 | |||
| 308 | logger.error(message, { response: listedObjects, ...lTags() }) | ||
| 309 | throw new Error(message) | ||
| 310 | } | ||
| 311 | |||
| 312 | await map(listedObjects.Contents, object => { | ||
| 313 | const command = commandBuilder(object) | ||
| 314 | |||
| 315 | return s3Client.send(command) | ||
| 316 | }, { concurrency: 10 }) | ||
| 317 | |||
| 318 | // Repeat if not all objects could be listed at once (limit of 1000?) | ||
| 319 | if (listedObjects.IsTruncated) { | ||
| 320 | await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) | ||
| 321 | } | ||
| 322 | } | ||
| 323 | |||
| 324 | function getACL (isPrivate: boolean) { | ||
| 325 | return isPrivate | ||
| 326 | ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE | ||
| 327 | : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC | ||
| 328 | } | ||
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts deleted file mode 100644 index 40619cd5a..000000000 --- a/server/lib/object-storage/urls.ts +++ /dev/null | |||
| @@ -1,63 +0,0 @@ | |||
| 1 | import { CONFIG } from '@server/initializers/config' | ||
| 2 | import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { MVideoUUID } from '@server/types/models' | ||
| 4 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' | ||
| 5 | |||
| 6 | function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { | ||
| 7 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) | ||
| 8 | } | ||
| 9 | |||
| 10 | // --------------------------------------------------------------------------- | ||
| 11 | |||
| 12 | function getWebVideoPublicFileUrl (fileUrl: string) { | ||
| 13 | const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL | ||
| 14 | if (!baseUrl) return fileUrl | ||
| 15 | |||
| 16 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
| 17 | } | ||
| 18 | |||
| 19 | function getHLSPublicFileUrl (fileUrl: string) { | ||
| 20 | const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL | ||
| 21 | if (!baseUrl) return fileUrl | ||
| 22 | |||
| 23 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
| 24 | } | ||
| 25 | |||
| 26 | // --------------------------------------------------------------------------- | ||
| 27 | |||
| 28 | function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { | ||
| 29 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` | ||
| 30 | } | ||
| 31 | |||
| 32 | function getWebVideoPrivateFileUrl (filename: string) { | ||
| 33 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename | ||
| 34 | } | ||
| 35 | |||
| 36 | // --------------------------------------------------------------------------- | ||
| 37 | |||
| 38 | export { | ||
| 39 | getInternalUrl, | ||
| 40 | |||
| 41 | getWebVideoPublicFileUrl, | ||
| 42 | getHLSPublicFileUrl, | ||
| 43 | |||
| 44 | getHLSPrivateFileUrl, | ||
| 45 | getWebVideoPrivateFileUrl, | ||
| 46 | |||
| 47 | replaceByBaseUrl | ||
| 48 | } | ||
| 49 | |||
| 50 | // --------------------------------------------------------------------------- | ||
| 51 | |||
| 52 | function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { | ||
| 53 | if (baseUrl) return baseUrl | ||
| 54 | |||
| 55 | return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` | ||
| 56 | } | ||
| 57 | |||
| 58 | const regex = new RegExp('https?://[^/]+') | ||
| 59 | function replaceByBaseUrl (fileUrl: string, baseUrl: string) { | ||
| 60 | if (!fileUrl) return fileUrl | ||
| 61 | |||
| 62 | return fileUrl.replace(regex, baseUrl) | ||
| 63 | } | ||
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts deleted file mode 100644 index 891e9ff76..000000000 --- a/server/lib/object-storage/videos.ts +++ /dev/null | |||
| @@ -1,197 +0,0 @@ | |||
| 1 | import { basename, join } from 'path' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { CONFIG } from '@server/initializers/config' | ||
| 4 | import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models' | ||
| 5 | import { getHLSDirectory } from '../paths' | ||
| 6 | import { VideoPathManager } from '../video-path-manager' | ||
| 7 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' | ||
| 8 | import { | ||
| 9 | createObjectReadStream, | ||
| 10 | listKeysOfPrefix, | ||
| 11 | lTags, | ||
| 12 | makeAvailable, | ||
| 13 | removeObject, | ||
| 14 | removeObjectByFullKey, | ||
| 15 | removePrefix, | ||
| 16 | storeContent, | ||
| 17 | storeObject, | ||
| 18 | updateObjectACL, | ||
| 19 | updatePrefixACL | ||
| 20 | } from './shared' | ||
| 21 | |||
| 22 | function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { | ||
| 23 | return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
| 24 | } | ||
| 25 | |||
| 26 | // --------------------------------------------------------------------------- | ||
| 27 | |||
| 28 | function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { | ||
| 29 | return storeObject({ | ||
| 30 | inputPath: join(getHLSDirectory(playlist.Video), filename), | ||
| 31 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), | ||
| 32 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
| 33 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
| 34 | }) | ||
| 35 | } | ||
| 36 | |||
| 37 | function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { | ||
| 38 | return storeObject({ | ||
| 39 | inputPath: path, | ||
| 40 | objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), | ||
| 41 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
| 42 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
| 43 | }) | ||
| 44 | } | ||
| 45 | |||
| 46 | function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { | ||
| 47 | return storeContent({ | ||
| 48 | content, | ||
| 49 | inputPath: path, | ||
| 50 | objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), | ||
| 51 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
| 52 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
| 53 | }) | ||
| 54 | } | ||
| 55 | |||
| 56 | // --------------------------------------------------------------------------- | ||
| 57 | |||
| 58 | function storeWebVideoFile (video: MVideo, file: MVideoFile) { | ||
| 59 | return storeObject({ | ||
| 60 | inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), | ||
| 61 | objectStorageKey: generateWebVideoObjectStorageKey(file.filename), | ||
| 62 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, | ||
| 63 | isPrivate: video.hasPrivateStaticPath() | ||
| 64 | }) | ||
| 65 | } | ||
| 66 | |||
| 67 | // --------------------------------------------------------------------------- | ||
| 68 | |||
| 69 | async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { | ||
| 70 | await updateObjectACL({ | ||
| 71 | objectStorageKey: generateWebVideoObjectStorageKey(file.filename), | ||
| 72 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, | ||
| 73 | isPrivate: video.hasPrivateStaticPath() | ||
| 74 | }) | ||
| 75 | } | ||
| 76 | |||
| 77 | async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { | ||
| 78 | await updatePrefixACL({ | ||
| 79 | prefix: generateHLSObjectBaseStorageKey(playlist), | ||
| 80 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
| 81 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
| 82 | }) | ||
| 83 | } | ||
| 84 | |||
| 85 | // --------------------------------------------------------------------------- | ||
| 86 | |||
| 87 | function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { | ||
| 88 | return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
| 89 | } | ||
| 90 | |||
| 91 | function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { | ||
| 92 | return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
| 93 | } | ||
| 94 | |||
| 95 | function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { | ||
| 96 | return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
| 97 | } | ||
| 98 | |||
| 99 | function removeHLSFileObjectStorageByFullKey (key: string) { | ||
| 100 | return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
| 101 | } | ||
| 102 | |||
| 103 | // --------------------------------------------------------------------------- | ||
| 104 | |||
| 105 | function removeWebVideoObjectStorage (videoFile: MVideoFile) { | ||
| 106 | return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS) | ||
| 107 | } | ||
| 108 | |||
| 109 | // --------------------------------------------------------------------------- | ||
| 110 | |||
| 111 | async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { | ||
| 112 | const key = generateHLSObjectStorageKey(playlist, filename) | ||
| 113 | |||
| 114 | logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) | ||
| 115 | |||
| 116 | await makeAvailable({ | ||
| 117 | key, | ||
| 118 | destination, | ||
| 119 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
| 120 | }) | ||
| 121 | |||
| 122 | return destination | ||
| 123 | } | ||
| 124 | |||
| 125 | async function makeWebVideoFileAvailable (filename: string, destination: string) { | ||
| 126 | const key = generateWebVideoObjectStorageKey(filename) | ||
| 127 | |||
| 128 | logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags()) | ||
| 129 | |||
| 130 | await makeAvailable({ | ||
| 131 | key, | ||
| 132 | destination, | ||
| 133 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS | ||
| 134 | }) | ||
| 135 | |||
| 136 | return destination | ||
| 137 | } | ||
| 138 | |||
| 139 | // --------------------------------------------------------------------------- | ||
| 140 | |||
| 141 | function getWebVideoFileReadStream (options: { | ||
| 142 | filename: string | ||
| 143 | rangeHeader: string | ||
| 144 | }) { | ||
| 145 | const { filename, rangeHeader } = options | ||
| 146 | |||
| 147 | const key = generateWebVideoObjectStorageKey(filename) | ||
| 148 | |||
| 149 | return createObjectReadStream({ | ||
| 150 | key, | ||
| 151 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, | ||
| 152 | rangeHeader | ||
| 153 | }) | ||
| 154 | } | ||
| 155 | |||
| 156 | function getHLSFileReadStream (options: { | ||
| 157 | playlist: MStreamingPlaylistVideo | ||
| 158 | filename: string | ||
| 159 | rangeHeader: string | ||
| 160 | }) { | ||
| 161 | const { playlist, filename, rangeHeader } = options | ||
| 162 | |||
| 163 | const key = generateHLSObjectStorageKey(playlist, filename) | ||
| 164 | |||
| 165 | return createObjectReadStream({ | ||
| 166 | key, | ||
| 167 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
| 168 | rangeHeader | ||
| 169 | }) | ||
| 170 | } | ||
| 171 | |||
| 172 | // --------------------------------------------------------------------------- | ||
| 173 | |||
| 174 | export { | ||
| 175 | listHLSFileKeysOf, | ||
| 176 | |||
| 177 | storeWebVideoFile, | ||
| 178 | storeHLSFileFromFilename, | ||
| 179 | storeHLSFileFromPath, | ||
| 180 | storeHLSFileFromContent, | ||
| 181 | |||
| 182 | updateWebVideoFileACL, | ||
| 183 | updateHLSFilesACL, | ||
| 184 | |||
| 185 | removeHLSObjectStorage, | ||
| 186 | removeHLSFileObjectStorageByFilename, | ||
| 187 | removeHLSFileObjectStorageByPath, | ||
| 188 | removeHLSFileObjectStorageByFullKey, | ||
| 189 | |||
| 190 | removeWebVideoObjectStorage, | ||
| 191 | |||
| 192 | makeWebVideoFileAvailable, | ||
| 193 | makeHLSFileAvailable, | ||
| 194 | |||
| 195 | getWebVideoFileReadStream, | ||
| 196 | getHLSFileReadStream | ||
| 197 | } | ||
diff --git a/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts deleted file mode 100644 index ef40c0fa9..000000000 --- a/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts +++ /dev/null | |||
| @@ -1,51 +0,0 @@ | |||
| 1 | import { Meter } from '@opentelemetry/api' | ||
| 2 | |||
| 3 | export class BittorrentTrackerObserversBuilder { | ||
| 4 | |||
| 5 | constructor (private readonly meter: Meter, private readonly trackerServer: any) { | ||
| 6 | |||
| 7 | } | ||
| 8 | |||
| 9 | buildObservers () { | ||
| 10 | const activeInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_active_infohashes_total', { | ||
| 11 | description: 'Total active infohashes in the PeerTube BitTorrent Tracker' | ||
| 12 | }) | ||
| 13 | const inactiveInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_inactive_infohashes_total', { | ||
| 14 | description: 'Total inactive infohashes in the PeerTube BitTorrent Tracker' | ||
| 15 | }) | ||
| 16 | const peers = this.meter.createObservableGauge('peertube_bittorrent_tracker_peers_total', { | ||
| 17 | description: 'Total peers in the PeerTube BitTorrent Tracker' | ||
| 18 | }) | ||
| 19 | |||
| 20 | this.meter.addBatchObservableCallback(observableResult => { | ||
| 21 | const infohashes = Object.keys(this.trackerServer.torrents) | ||
| 22 | |||
| 23 | const counters = { | ||
| 24 | activeInfohashes: 0, | ||
| 25 | inactiveInfohashes: 0, | ||
| 26 | peers: 0, | ||
| 27 | uncompletedPeers: 0 | ||
| 28 | } | ||
| 29 | |||
| 30 | for (const infohash of infohashes) { | ||
| 31 | const content = this.trackerServer.torrents[infohash] | ||
| 32 | |||
| 33 | const peers = content.peers | ||
| 34 | if (peers.keys.length !== 0) counters.activeInfohashes++ | ||
| 35 | else counters.inactiveInfohashes++ | ||
| 36 | |||
| 37 | for (const peerId of peers.keys) { | ||
| 38 | const peer = peers.peek(peerId) | ||
| 39 | if (peer == null) return | ||
| 40 | |||
| 41 | counters.peers++ | ||
| 42 | } | ||
| 43 | } | ||
| 44 | |||
| 45 | observableResult.observe(activeInfohashes, counters.activeInfohashes) | ||
| 46 | observableResult.observe(inactiveInfohashes, counters.inactiveInfohashes) | ||
| 47 | observableResult.observe(peers, counters.peers) | ||
| 48 | }, [ activeInfohashes, inactiveInfohashes, peers ]) | ||
| 49 | } | ||
| 50 | |||
| 51 | } | ||
diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts deleted file mode 100644 index 47b24a54f..000000000 --- a/server/lib/opentelemetry/metric-helpers/index.ts +++ /dev/null | |||
| @@ -1,7 +0,0 @@ | |||
| 1 | export * from './bittorrent-tracker-observers-builder' | ||
| 2 | export * from './lives-observers-builder' | ||
| 3 | export * from './job-queue-observers-builder' | ||
| 4 | export * from './nodejs-observers-builder' | ||
| 5 | export * from './playback-metrics' | ||
| 6 | export * from './stats-observers-builder' | ||
| 7 | export * from './viewers-observers-builder' | ||
diff --git a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts deleted file mode 100644 index 56713ede8..000000000 --- a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts +++ /dev/null | |||
| @@ -1,24 +0,0 @@ | |||
| 1 | import { Meter } from '@opentelemetry/api' | ||
| 2 | import { JobQueue } from '@server/lib/job-queue' | ||
| 3 | |||
| 4 | export class JobQueueObserversBuilder { | ||
| 5 | |||
| 6 | constructor (private readonly meter: Meter) { | ||
| 7 | |||
| 8 | } | ||
| 9 | |||
| 10 | buildObservers () { | ||
| 11 | this.meter.createObservableGauge('peertube_job_queue_total', { | ||
| 12 | description: 'Total jobs in the PeerTube job queue' | ||
| 13 | }).addCallback(async observableResult => { | ||
| 14 | const stats = await JobQueue.Instance.getStats() | ||
| 15 | |||
| 16 | for (const { jobType, counts } of stats) { | ||
| 17 | for (const state of Object.keys(counts)) { | ||
| 18 | observableResult.observe(counts[state], { jobType, state }) | ||
| 19 | } | ||
| 20 | } | ||
| 21 | }) | ||
| 22 | } | ||
| 23 | |||
| 24 | } | ||
diff --git a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts deleted file mode 100644 index 5effc18e1..000000000 --- a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts +++ /dev/null | |||
| @@ -1,21 +0,0 @@ | |||
| 1 | import { Meter } from '@opentelemetry/api' | ||
| 2 | import { VideoModel } from '@server/models/video/video' | ||
| 3 | |||
| 4 | export class LivesObserversBuilder { | ||
| 5 | |||
| 6 | constructor (private readonly meter: Meter) { | ||
| 7 | |||
| 8 | } | ||
| 9 | |||
| 10 | buildObservers () { | ||
| 11 | this.meter.createObservableGauge('peertube_running_lives_total', { | ||
| 12 | description: 'Total running lives on the instance' | ||
| 13 | }).addCallback(async observableResult => { | ||
| 14 | const local = await VideoModel.countLives({ remote: false, mode: 'published' }) | ||
| 15 | const remote = await VideoModel.countLives({ remote: true, mode: 'published' }) | ||
| 16 | |||
| 17 | observableResult.observe(local, { liveOrigin: 'local' }) | ||
| 18 | observableResult.observe(remote, { liveOrigin: 'remote' }) | ||
| 19 | }) | ||
| 20 | } | ||
| 21 | } | ||
diff --git a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts deleted file mode 100644 index 8ed219e9e..000000000 --- a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts +++ /dev/null | |||
| @@ -1,202 +0,0 @@ | |||
| 1 | import { readdir } from 'fs-extra' | ||
| 2 | import { constants, NodeGCPerformanceDetail, PerformanceObserver } from 'perf_hooks' | ||
| 3 | import * as process from 'process' | ||
| 4 | import { Meter, ObservableResult } from '@opentelemetry/api' | ||
| 5 | import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics' | ||
| 6 | import { View } from '@opentelemetry/sdk-metrics/build/src/view/View' | ||
| 7 | import { logger } from '@server/helpers/logger' | ||
| 8 | |||
| 9 | // Thanks to https://github.com/siimon/prom-client | ||
| 10 | // We took their logic and adapter it for opentelemetry | ||
| 11 | // Try to keep consistency with their metric name/description so it's easier to process (grafana dashboard template etc) | ||
| 12 | |||
| 13 | export class NodeJSObserversBuilder { | ||
| 14 | |||
| 15 | constructor (private readonly meter: Meter) { | ||
| 16 | } | ||
| 17 | |||
| 18 | static getViews () { | ||
| 19 | return [ | ||
| 20 | new View({ | ||
| 21 | aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]), | ||
| 22 | instrumentName: 'nodejs_gc_duration_seconds' | ||
| 23 | }) | ||
| 24 | ] | ||
| 25 | } | ||
| 26 | |||
| 27 | buildObservers () { | ||
| 28 | this.buildCPUObserver() | ||
| 29 | this.buildMemoryObserver() | ||
| 30 | |||
| 31 | this.buildHandlesObserver() | ||
| 32 | this.buildFileDescriptorsObserver() | ||
| 33 | |||
| 34 | this.buildGCObserver() | ||
| 35 | this.buildEventLoopLagObserver() | ||
| 36 | |||
| 37 | this.buildLibUVActiveRequestsObserver() | ||
| 38 | this.buildActiveResourcesObserver() | ||
| 39 | } | ||
| 40 | |||
| 41 | private buildCPUObserver () { | ||
| 42 | const cpuTotal = this.meter.createObservableCounter('process_cpu_seconds_total', { | ||
| 43 | description: 'Total user and system CPU time spent in seconds.' | ||
| 44 | }) | ||
| 45 | const cpuUser = this.meter.createObservableCounter('process_cpu_user_seconds_total', { | ||
| 46 | description: 'Total user CPU time spent in seconds.' | ||
| 47 | }) | ||
| 48 | const cpuSystem = this.meter.createObservableCounter('process_cpu_system_seconds_total', { | ||
| 49 | description: 'Total system CPU time spent in seconds.' | ||
| 50 | }) | ||
| 51 | |||
| 52 | let lastCpuUsage = process.cpuUsage() | ||
| 53 | |||
| 54 | this.meter.addBatchObservableCallback(observableResult => { | ||
| 55 | const cpuUsage = process.cpuUsage() | ||
| 56 | |||
| 57 | const userUsageMicros = cpuUsage.user - lastCpuUsage.user | ||
| 58 | const systemUsageMicros = cpuUsage.system - lastCpuUsage.system | ||
| 59 | |||
| 60 | lastCpuUsage = cpuUsage | ||
| 61 | |||
| 62 | observableResult.observe(cpuTotal, (userUsageMicros + systemUsageMicros) / 1e6) | ||
| 63 | observableResult.observe(cpuUser, userUsageMicros / 1e6) | ||
| 64 | observableResult.observe(cpuSystem, systemUsageMicros / 1e6) | ||
| 65 | |||
| 66 | }, [ cpuTotal, cpuUser, cpuSystem ]) | ||
| 67 | } | ||
| 68 | |||
| 69 | private buildMemoryObserver () { | ||
| 70 | this.meter.createObservableGauge('nodejs_memory_usage_bytes', { | ||
| 71 | description: 'Memory' | ||
| 72 | }).addCallback(observableResult => { | ||
| 73 | const current = process.memoryUsage() | ||
| 74 | |||
| 75 | observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' }) | ||
| 76 | observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' }) | ||
| 77 | observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' }) | ||
| 78 | observableResult.observe(current.external, { memoryType: 'external' }) | ||
| 79 | observableResult.observe(current.rss, { memoryType: 'rss' }) | ||
| 80 | }) | ||
| 81 | } | ||
| 82 | |||
| 83 | private buildHandlesObserver () { | ||
| 84 | if (typeof (process as any)._getActiveHandles !== 'function') return | ||
| 85 | |||
| 86 | this.meter.createObservableGauge('nodejs_active_handles_total', { | ||
| 87 | description: 'Total number of active handles.' | ||
| 88 | }).addCallback(observableResult => { | ||
| 89 | const handles = (process as any)._getActiveHandles() | ||
| 90 | |||
| 91 | observableResult.observe(handles.length) | ||
| 92 | }) | ||
| 93 | } | ||
| 94 | |||
| 95 | private buildGCObserver () { | ||
| 96 | const kinds = { | ||
| 97 | [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major', | ||
| 98 | [constants.NODE_PERFORMANCE_GC_MINOR]: 'minor', | ||
| 99 | [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental', | ||
| 100 | [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb' | ||
| 101 | } | ||
| 102 | |||
| 103 | const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', { | ||
| 104 | description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb' | ||
| 105 | }) | ||
| 106 | |||
| 107 | const obs = new PerformanceObserver(list => { | ||
| 108 | const entry = list.getEntries()[0] | ||
| 109 | |||
| 110 | // Node < 16 uses entry.kind | ||
| 111 | // Node >= 16 uses entry.detail.kind | ||
| 112 | // See: https://nodejs.org/docs/latest-v16.x/api/deprecations.html#deprecations_dep0152_extension_performanceentry_properties | ||
| 113 | const kind = entry.detail | ||
| 114 | ? kinds[(entry.detail as NodeGCPerformanceDetail).kind] | ||
| 115 | : kinds[(entry as any).kind] | ||
| 116 | |||
| 117 | // Convert duration from milliseconds to seconds | ||
| 118 | histogram.record(entry.duration / 1000, { | ||
| 119 | kind | ||
| 120 | }) | ||
| 121 | }) | ||
| 122 | |||
| 123 | obs.observe({ entryTypes: [ 'gc' ] }) | ||
| 124 | } | ||
| 125 | |||
| 126 | private buildEventLoopLagObserver () { | ||
| 127 | const reportEventloopLag = (start: [ number, number ], observableResult: ObservableResult, res: () => void) => { | ||
| 128 | const delta = process.hrtime(start) | ||
| 129 | const nanosec = delta[0] * 1e9 + delta[1] | ||
| 130 | const seconds = nanosec / 1e9 | ||
| 131 | |||
| 132 | observableResult.observe(seconds) | ||
| 133 | |||
| 134 | res() | ||
| 135 | } | ||
| 136 | |||
| 137 | this.meter.createObservableGauge('nodejs_eventloop_lag_seconds', { | ||
| 138 | description: 'Lag of event loop in seconds.' | ||
| 139 | }).addCallback(observableResult => { | ||
| 140 | return new Promise(res => { | ||
| 141 | const start = process.hrtime() | ||
| 142 | |||
| 143 | setImmediate(reportEventloopLag, start, observableResult, res) | ||
| 144 | }) | ||
| 145 | }) | ||
| 146 | } | ||
| 147 | |||
| 148 | private buildFileDescriptorsObserver () { | ||
| 149 | this.meter.createObservableGauge('process_open_fds', { | ||
| 150 | description: 'Number of open file descriptors.' | ||
| 151 | }).addCallback(async observableResult => { | ||
| 152 | try { | ||
| 153 | const fds = await readdir('/proc/self/fd') | ||
| 154 | observableResult.observe(fds.length - 1) | ||
| 155 | } catch (err) { | ||
| 156 | logger.debug('Cannot list file descriptors of current process for OpenTelemetry.', { err }) | ||
| 157 | } | ||
| 158 | }) | ||
| 159 | } | ||
| 160 | |||
| 161 | private buildLibUVActiveRequestsObserver () { | ||
| 162 | if (typeof (process as any)._getActiveRequests !== 'function') return | ||
| 163 | |||
| 164 | this.meter.createObservableGauge('nodejs_active_requests_total', { | ||
| 165 | description: 'Total number of active libuv requests.' | ||
| 166 | }).addCallback(observableResult => { | ||
| 167 | const requests = (process as any)._getActiveRequests() | ||
| 168 | |||
| 169 | observableResult.observe(requests.length) | ||
| 170 | }) | ||
| 171 | } | ||
| 172 | |||
| 173 | private buildActiveResourcesObserver () { | ||
| 174 | if (typeof (process as any).getActiveResourcesInfo !== 'function') return | ||
| 175 | |||
| 176 | const grouped = this.meter.createObservableCounter('nodejs_active_resources', { | ||
| 177 | description: 'Number of active resources that are currently keeping the event loop alive, grouped by async resource type.' | ||
| 178 | }) | ||
| 179 | const total = this.meter.createObservableCounter('nodejs_active_resources_total', { | ||
| 180 | description: 'Total number of active resources.' | ||
| 181 | }) | ||
| 182 | |||
| 183 | this.meter.addBatchObservableCallback(observableResult => { | ||
| 184 | const resources = (process as any).getActiveResourcesInfo() | ||
| 185 | |||
| 186 | const data = {} | ||
| 187 | |||
| 188 | for (let i = 0; i < resources.length; i++) { | ||
| 189 | const resource = resources[i] | ||
| 190 | |||
| 191 | if (data[resource] === undefined) data[resource] = 0 | ||
| 192 | data[resource] += 1 | ||
| 193 | } | ||
| 194 | |||
| 195 | for (const type of Object.keys(data)) { | ||
| 196 | observableResult.observe(grouped, data[type], { type }) | ||
| 197 | } | ||
| 198 | |||
| 199 | observableResult.observe(total, resources.length) | ||
| 200 | }, [ grouped, total ]) | ||
| 201 | } | ||
| 202 | } | ||
diff --git a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts deleted file mode 100644 index 1eb08b5a6..000000000 --- a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts +++ /dev/null | |||
| @@ -1,85 +0,0 @@ | |||
| 1 | import { Counter, Meter } from '@opentelemetry/api' | ||
| 2 | import { MVideoImmutable } from '@server/types/models' | ||
| 3 | import { PlaybackMetricCreate } from '@shared/models' | ||
| 4 | |||
| 5 | export class PlaybackMetrics { | ||
| 6 | private errorsCounter: Counter | ||
| 7 | private resolutionChangesCounter: Counter | ||
| 8 | |||
| 9 | private downloadedBytesP2PCounter: Counter | ||
| 10 | private uploadedBytesP2PCounter: Counter | ||
| 11 | |||
| 12 | private downloadedBytesHTTPCounter: Counter | ||
| 13 | |||
| 14 | private peersP2PPeersGaugeBuffer: { | ||
| 15 | value: number | ||
| 16 | attributes: any | ||
| 17 | }[] = [] | ||
| 18 | |||
| 19 | constructor (private readonly meter: Meter) { | ||
| 20 | |||
| 21 | } | ||
| 22 | |||
| 23 | buildCounters () { | ||
| 24 | this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', { | ||
| 25 | description: 'Errors collected from PeerTube player.' | ||
| 26 | }) | ||
| 27 | |||
| 28 | this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', { | ||
| 29 | description: 'Resolution changes collected from PeerTube player.' | ||
| 30 | }) | ||
| 31 | |||
| 32 | this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', { | ||
| 33 | description: 'Downloaded bytes with HTTP by PeerTube player.' | ||
| 34 | }) | ||
| 35 | this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', { | ||
| 36 | description: 'Downloaded bytes with P2P by PeerTube player.' | ||
| 37 | }) | ||
| 38 | |||
| 39 | this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', { | ||
| 40 | description: 'Uploaded bytes with P2P by PeerTube player.' | ||
| 41 | }) | ||
| 42 | |||
| 43 | this.meter.createObservableGauge('peertube_playback_p2p_peers', { | ||
| 44 | description: 'Total P2P peers connected to the PeerTube player.' | ||
| 45 | }).addCallback(observableResult => { | ||
| 46 | for (const gauge of this.peersP2PPeersGaugeBuffer) { | ||
| 47 | observableResult.observe(gauge.value, gauge.attributes) | ||
| 48 | } | ||
| 49 | |||
| 50 | this.peersP2PPeersGaugeBuffer = [] | ||
| 51 | }) | ||
| 52 | } | ||
| 53 | |||
| 54 | observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) { | ||
| 55 | const attributes = { | ||
| 56 | videoOrigin: video.remote | ||
| 57 | ? 'remote' | ||
| 58 | : 'local', | ||
| 59 | |||
| 60 | playerMode: metrics.playerMode, | ||
| 61 | |||
| 62 | resolution: metrics.resolution + '', | ||
| 63 | fps: metrics.fps + '', | ||
| 64 | |||
| 65 | p2pEnabled: metrics.p2pEnabled, | ||
| 66 | |||
| 67 | videoUUID: video.uuid | ||
| 68 | } | ||
| 69 | |||
| 70 | this.errorsCounter.add(metrics.errors, attributes) | ||
| 71 | this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes) | ||
| 72 | |||
| 73 | this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes) | ||
| 74 | this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes) | ||
| 75 | |||
| 76 | this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes) | ||
| 77 | |||
| 78 | if (metrics.p2pPeers) { | ||
| 79 | this.peersP2PPeersGaugeBuffer.push({ | ||
| 80 | value: metrics.p2pPeers, | ||
| 81 | attributes | ||
| 82 | }) | ||
| 83 | } | ||
| 84 | } | ||
| 85 | } | ||
diff --git a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts deleted file mode 100644 index 9f5f22e1b..000000000 --- a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts +++ /dev/null | |||
| @@ -1,186 +0,0 @@ | |||
| 1 | import memoizee from 'memoizee' | ||
| 2 | import { Meter } from '@opentelemetry/api' | ||
| 3 | import { MEMOIZE_TTL } from '@server/initializers/constants' | ||
| 4 | import { buildAvailableActivities } from '@server/lib/activitypub/activity' | ||
| 5 | import { StatsManager } from '@server/lib/stat-manager' | ||
| 6 | |||
| 7 | export class StatsObserversBuilder { | ||
| 8 | |||
| 9 | private readonly getInstanceStats = memoizee(() => { | ||
| 10 | return StatsManager.Instance.getStats() | ||
| 11 | }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS }) | ||
| 12 | |||
| 13 | constructor (private readonly meter: Meter) { | ||
| 14 | |||
| 15 | } | ||
| 16 | |||
| 17 | buildObservers () { | ||
| 18 | this.buildUserStatsObserver() | ||
| 19 | this.buildVideoStatsObserver() | ||
| 20 | this.buildCommentStatsObserver() | ||
| 21 | this.buildPlaylistStatsObserver() | ||
| 22 | this.buildChannelStatsObserver() | ||
| 23 | this.buildInstanceFollowsStatsObserver() | ||
| 24 | this.buildRedundancyStatsObserver() | ||
| 25 | this.buildActivityPubStatsObserver() | ||
| 26 | } | ||
| 27 | |||
| 28 | private buildUserStatsObserver () { | ||
| 29 | this.meter.createObservableGauge('peertube_users_total', { | ||
| 30 | description: 'Total users on the instance' | ||
| 31 | }).addCallback(async observableResult => { | ||
| 32 | const stats = await this.getInstanceStats() | ||
| 33 | |||
| 34 | observableResult.observe(stats.totalUsers) | ||
| 35 | }) | ||
| 36 | |||
| 37 | this.meter.createObservableGauge('peertube_active_users_total', { | ||
| 38 | description: 'Total active users on the instance' | ||
| 39 | }).addCallback(async observableResult => { | ||
| 40 | const stats = await this.getInstanceStats() | ||
| 41 | |||
| 42 | observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' }) | ||
| 43 | observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' }) | ||
| 44 | observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' }) | ||
| 45 | }) | ||
| 46 | } | ||
| 47 | |||
| 48 | private buildChannelStatsObserver () { | ||
| 49 | this.meter.createObservableGauge('peertube_channels_total', { | ||
| 50 | description: 'Total channels on the instance' | ||
| 51 | }).addCallback(async observableResult => { | ||
| 52 | const stats = await this.getInstanceStats() | ||
| 53 | |||
| 54 | observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' }) | ||
| 55 | }) | ||
| 56 | |||
| 57 | this.meter.createObservableGauge('peertube_active_channels_total', { | ||
| 58 | description: 'Total active channels on the instance' | ||
| 59 | }).addCallback(async observableResult => { | ||
| 60 | const stats = await this.getInstanceStats() | ||
| 61 | |||
| 62 | observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' }) | ||
| 63 | observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' }) | ||
| 64 | observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' }) | ||
| 65 | }) | ||
| 66 | } | ||
| 67 | |||
| 68 | private buildVideoStatsObserver () { | ||
| 69 | this.meter.createObservableGauge('peertube_videos_total', { | ||
| 70 | description: 'Total videos on the instance' | ||
| 71 | }).addCallback(async observableResult => { | ||
| 72 | const stats = await this.getInstanceStats() | ||
| 73 | |||
| 74 | observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' }) | ||
| 75 | observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' }) | ||
| 76 | }) | ||
| 77 | |||
| 78 | this.meter.createObservableGauge('peertube_video_views_total', { | ||
| 79 | description: 'Total video views made on the instance' | ||
| 80 | }).addCallback(async observableResult => { | ||
| 81 | const stats = await this.getInstanceStats() | ||
| 82 | |||
| 83 | observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' }) | ||
| 84 | }) | ||
| 85 | |||
| 86 | this.meter.createObservableGauge('peertube_video_bytes_total', { | ||
| 87 | description: 'Total bytes of videos' | ||
| 88 | }).addCallback(async observableResult => { | ||
| 89 | const stats = await this.getInstanceStats() | ||
| 90 | |||
| 91 | observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' }) | ||
| 92 | }) | ||
| 93 | } | ||
| 94 | |||
| 95 | private buildCommentStatsObserver () { | ||
| 96 | this.meter.createObservableGauge('peertube_comments_total', { | ||
| 97 | description: 'Total comments on the instance' | ||
| 98 | }).addCallback(async observableResult => { | ||
| 99 | const stats = await this.getInstanceStats() | ||
| 100 | |||
| 101 | observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' }) | ||
| 102 | }) | ||
| 103 | } | ||
| 104 | |||
| 105 | private buildPlaylistStatsObserver () { | ||
| 106 | this.meter.createObservableGauge('peertube_playlists_total', { | ||
| 107 | description: 'Total playlists on the instance' | ||
| 108 | }).addCallback(async observableResult => { | ||
| 109 | const stats = await this.getInstanceStats() | ||
| 110 | |||
| 111 | observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' }) | ||
| 112 | }) | ||
| 113 | } | ||
| 114 | |||
| 115 | private buildInstanceFollowsStatsObserver () { | ||
| 116 | this.meter.createObservableGauge('peertube_instance_followers_total', { | ||
| 117 | description: 'Total followers of the instance' | ||
| 118 | }).addCallback(async observableResult => { | ||
| 119 | const stats = await this.getInstanceStats() | ||
| 120 | |||
| 121 | observableResult.observe(stats.totalInstanceFollowers) | ||
| 122 | }) | ||
| 123 | |||
| 124 | this.meter.createObservableGauge('peertube_instance_following_total', { | ||
| 125 | description: 'Total following of the instance' | ||
| 126 | }).addCallback(async observableResult => { | ||
| 127 | const stats = await this.getInstanceStats() | ||
| 128 | |||
| 129 | observableResult.observe(stats.totalInstanceFollowing) | ||
| 130 | }) | ||
| 131 | } | ||
| 132 | |||
| 133 | private buildRedundancyStatsObserver () { | ||
| 134 | this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', { | ||
| 135 | description: 'Total redundancy used of the instance' | ||
| 136 | }).addCallback(async observableResult => { | ||
| 137 | const stats = await this.getInstanceStats() | ||
| 138 | |||
| 139 | for (const r of stats.videosRedundancy) { | ||
| 140 | observableResult.observe(r.totalUsed, { strategy: r.strategy }) | ||
| 141 | } | ||
| 142 | }) | ||
| 143 | |||
| 144 | this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', { | ||
| 145 | description: 'Total redundancy available of the instance' | ||
| 146 | }).addCallback(async observableResult => { | ||
| 147 | const stats = await this.getInstanceStats() | ||
| 148 | |||
| 149 | for (const r of stats.videosRedundancy) { | ||
| 150 | observableResult.observe(r.totalSize, { strategy: r.strategy }) | ||
| 151 | } | ||
| 152 | }) | ||
| 153 | } | ||
| 154 | |||
| 155 | private buildActivityPubStatsObserver () { | ||
| 156 | const availableActivities = buildAvailableActivities() | ||
| 157 | |||
| 158 | this.meter.createObservableGauge('peertube_ap_inbox_success_total', { | ||
| 159 | description: 'Total inbox messages processed with success' | ||
| 160 | }).addCallback(async observableResult => { | ||
| 161 | const stats = await this.getInstanceStats() | ||
| 162 | |||
| 163 | for (const type of availableActivities) { | ||
| 164 | observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type }) | ||
| 165 | } | ||
| 166 | }) | ||
| 167 | |||
| 168 | this.meter.createObservableGauge('peertube_ap_inbox_error_total', { | ||
| 169 | description: 'Total inbox messages processed with error' | ||
| 170 | }).addCallback(async observableResult => { | ||
| 171 | const stats = await this.getInstanceStats() | ||
| 172 | |||
| 173 | for (const type of availableActivities) { | ||
| 174 | observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type }) | ||
| 175 | } | ||
| 176 | }) | ||
| 177 | |||
| 178 | this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', { | ||
| 179 | description: 'Total inbox messages waiting for being processed' | ||
| 180 | }).addCallback(async observableResult => { | ||
| 181 | const stats = await this.getInstanceStats() | ||
| 182 | |||
| 183 | observableResult.observe(stats.totalActivityPubMessagesWaiting) | ||
| 184 | }) | ||
| 185 | } | ||
| 186 | } | ||
diff --git a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts deleted file mode 100644 index c65f8ddae..000000000 --- a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts +++ /dev/null | |||
| @@ -1,24 +0,0 @@ | |||
| 1 | import { Meter } from '@opentelemetry/api' | ||
| 2 | import { VideoScope, ViewerScope } from '@server/lib/views/shared' | ||
| 3 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' | ||
| 4 | |||
| 5 | export class ViewersObserversBuilder { | ||
| 6 | |||
| 7 | constructor (private readonly meter: Meter) { | ||
| 8 | |||
| 9 | } | ||
| 10 | |||
| 11 | buildObservers () { | ||
| 12 | this.meter.createObservableGauge('peertube_viewers_total', { | ||
| 13 | description: 'Total viewers on the instance' | ||
| 14 | }).addCallback(observableResult => { | ||
| 15 | for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) { | ||
| 16 | for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) { | ||
| 17 | const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope }) | ||
| 18 | |||
| 19 | observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope }) | ||
| 20 | } | ||
| 21 | } | ||
| 22 | }) | ||
| 23 | } | ||
| 24 | } | ||
diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts deleted file mode 100644 index bffe00840..000000000 --- a/server/lib/opentelemetry/metrics.ts +++ /dev/null | |||
| @@ -1,123 +0,0 @@ | |||
| 1 | import { Application, Request, Response } from 'express' | ||
| 2 | import { Meter, metrics } from '@opentelemetry/api' | ||
| 3 | import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' | ||
| 4 | import { MeterProvider } from '@opentelemetry/sdk-metrics' | ||
| 5 | import { logger } from '@server/helpers/logger' | ||
| 6 | import { CONFIG } from '@server/initializers/config' | ||
| 7 | import { MVideoImmutable } from '@server/types/models' | ||
| 8 | import { PlaybackMetricCreate } from '@shared/models' | ||
| 9 | import { | ||
| 10 | BittorrentTrackerObserversBuilder, | ||
| 11 | JobQueueObserversBuilder, | ||
| 12 | LivesObserversBuilder, | ||
| 13 | NodeJSObserversBuilder, | ||
| 14 | PlaybackMetrics, | ||
| 15 | StatsObserversBuilder, | ||
| 16 | ViewersObserversBuilder | ||
| 17 | } from './metric-helpers' | ||
| 18 | |||
| 19 | class OpenTelemetryMetrics { | ||
| 20 | |||
| 21 | private static instance: OpenTelemetryMetrics | ||
| 22 | |||
| 23 | private meter: Meter | ||
| 24 | |||
| 25 | private onRequestDuration: (req: Request, res: Response) => void | ||
| 26 | |||
| 27 | private playbackMetrics: PlaybackMetrics | ||
| 28 | |||
| 29 | private constructor () {} | ||
| 30 | |||
| 31 | init (app: Application) { | ||
| 32 | if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return | ||
| 33 | |||
| 34 | app.use((req, res, next) => { | ||
| 35 | res.once('finish', () => { | ||
| 36 | if (!this.onRequestDuration) return | ||
| 37 | |||
| 38 | this.onRequestDuration(req as Request, res as Response) | ||
| 39 | }) | ||
| 40 | |||
| 41 | next() | ||
| 42 | }) | ||
| 43 | } | ||
| 44 | |||
| 45 | registerMetrics (options: { trackerServer: any }) { | ||
| 46 | if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return | ||
| 47 | |||
| 48 | logger.info('Registering Open Telemetry metrics') | ||
| 49 | |||
| 50 | const provider = new MeterProvider({ | ||
| 51 | views: [ | ||
| 52 | ...NodeJSObserversBuilder.getViews() | ||
| 53 | ] | ||
| 54 | }) | ||
| 55 | |||
| 56 | provider.addMetricReader(new PrometheusExporter({ | ||
| 57 | host: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.HOSTNAME, | ||
| 58 | port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT | ||
| 59 | })) | ||
| 60 | |||
| 61 | metrics.setGlobalMeterProvider(provider) | ||
| 62 | |||
| 63 | this.meter = metrics.getMeter('default') | ||
| 64 | |||
| 65 | if (CONFIG.OPEN_TELEMETRY.METRICS.HTTP_REQUEST_DURATION.ENABLED === true) { | ||
| 66 | this.buildRequestObserver() | ||
| 67 | } | ||
| 68 | |||
| 69 | this.playbackMetrics = new PlaybackMetrics(this.meter) | ||
| 70 | this.playbackMetrics.buildCounters() | ||
| 71 | |||
| 72 | const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter) | ||
| 73 | nodeJSObserversBuilder.buildObservers() | ||
| 74 | |||
| 75 | const jobQueueObserversBuilder = new JobQueueObserversBuilder(this.meter) | ||
| 76 | jobQueueObserversBuilder.buildObservers() | ||
| 77 | |||
| 78 | const statsObserversBuilder = new StatsObserversBuilder(this.meter) | ||
| 79 | statsObserversBuilder.buildObservers() | ||
| 80 | |||
| 81 | const livesObserversBuilder = new LivesObserversBuilder(this.meter) | ||
| 82 | livesObserversBuilder.buildObservers() | ||
| 83 | |||
| 84 | const viewersObserversBuilder = new ViewersObserversBuilder(this.meter) | ||
| 85 | viewersObserversBuilder.buildObservers() | ||
| 86 | |||
| 87 | const bittorrentTrackerObserversBuilder = new BittorrentTrackerObserversBuilder(this.meter, options.trackerServer) | ||
| 88 | bittorrentTrackerObserversBuilder.buildObservers() | ||
| 89 | } | ||
| 90 | |||
| 91 | observePlaybackMetric (video: MVideoImmutable, metrics: PlaybackMetricCreate) { | ||
| 92 | this.playbackMetrics.observe(video, metrics) | ||
| 93 | } | ||
| 94 | |||
| 95 | private buildRequestObserver () { | ||
| 96 | const requestDuration = this.meter.createHistogram('http_request_duration_ms', { | ||
| 97 | unit: 'milliseconds', | ||
| 98 | description: 'Duration of HTTP requests in ms' | ||
| 99 | }) | ||
| 100 | |||
| 101 | this.onRequestDuration = (req: Request, res: Response) => { | ||
| 102 | const duration = Date.now() - res.locals.requestStart | ||
| 103 | |||
| 104 | requestDuration.record(duration, { | ||
| 105 | path: this.buildRequestPath(req.originalUrl), | ||
| 106 | method: req.method, | ||
| 107 | statusCode: res.statusCode + '' | ||
| 108 | }) | ||
| 109 | } | ||
| 110 | } | ||
| 111 | |||
| 112 | private buildRequestPath (path: string) { | ||
| 113 | return path.split('?')[0] | ||
| 114 | } | ||
| 115 | |||
| 116 | static get Instance () { | ||
| 117 | return this.instance || (this.instance = new this()) | ||
| 118 | } | ||
| 119 | } | ||
| 120 | |||
| 121 | export { | ||
| 122 | OpenTelemetryMetrics | ||
| 123 | } | ||
diff --git a/server/lib/opentelemetry/tracing.ts b/server/lib/opentelemetry/tracing.ts deleted file mode 100644 index 9a81680b2..000000000 --- a/server/lib/opentelemetry/tracing.ts +++ /dev/null | |||
| @@ -1,94 +0,0 @@ | |||
| 1 | import { SequelizeInstrumentation } from 'opentelemetry-instrumentation-sequelize' | ||
| 2 | import { context, diag, DiagLogLevel, trace } from '@opentelemetry/api' | ||
| 3 | import { JaegerExporter } from '@opentelemetry/exporter-jaeger' | ||
| 4 | import { registerInstrumentations } from '@opentelemetry/instrumentation' | ||
| 5 | import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns' | ||
| 6 | import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express' | ||
| 7 | import FsInstrumentation from '@opentelemetry/instrumentation-fs' | ||
| 8 | import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' | ||
| 9 | import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis' | ||
| 10 | import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' | ||
| 11 | import { Resource } from '@opentelemetry/resources' | ||
| 12 | import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' | ||
| 13 | import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' | ||
| 14 | import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' | ||
| 15 | import { logger } from '@server/helpers/logger' | ||
| 16 | import { CONFIG } from '@server/initializers/config' | ||
| 17 | |||
| 18 | const tracer = trace.getTracer('peertube') | ||
| 19 | |||
| 20 | function registerOpentelemetryTracing () { | ||
| 21 | if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) return | ||
| 22 | |||
| 23 | logger.info('Registering Open Telemetry tracing') | ||
| 24 | |||
| 25 | const customLogger = (level: string) => { | ||
| 26 | return (message: string, ...args: unknown[]) => { | ||
| 27 | let fullMessage = message | ||
| 28 | |||
| 29 | for (const arg of args) { | ||
| 30 | if (typeof arg === 'string') fullMessage += arg | ||
| 31 | else break | ||
| 32 | } | ||
| 33 | |||
| 34 | logger[level](fullMessage) | ||
| 35 | } | ||
| 36 | } | ||
| 37 | |||
| 38 | diag.setLogger({ | ||
| 39 | error: customLogger('error'), | ||
| 40 | warn: customLogger('warn'), | ||
| 41 | info: customLogger('info'), | ||
| 42 | debug: customLogger('debug'), | ||
| 43 | verbose: customLogger('verbose') | ||
| 44 | }, DiagLogLevel.INFO) | ||
| 45 | |||
| 46 | const tracerProvider = new NodeTracerProvider({ | ||
| 47 | resource: new Resource({ | ||
| 48 | [SemanticResourceAttributes.SERVICE_NAME]: 'peertube' | ||
| 49 | }) | ||
| 50 | }) | ||
| 51 | |||
| 52 | registerInstrumentations({ | ||
| 53 | tracerProvider, | ||
| 54 | instrumentations: [ | ||
| 55 | new PgInstrumentation({ | ||
| 56 | enhancedDatabaseReporting: true | ||
| 57 | }), | ||
| 58 | new DnsInstrumentation(), | ||
| 59 | new HttpInstrumentation(), | ||
| 60 | new ExpressInstrumentation(), | ||
| 61 | new IORedisInstrumentation({ | ||
| 62 | dbStatementSerializer: function (cmdName, cmdArgs) { | ||
| 63 | return [ cmdName, ...cmdArgs ].join(' ') | ||
| 64 | } | ||
| 65 | }), | ||
| 66 | new FsInstrumentation(), | ||
| 67 | new SequelizeInstrumentation() | ||
| 68 | ] | ||
| 69 | }) | ||
| 70 | |||
| 71 | tracerProvider.addSpanProcessor( | ||
| 72 | new BatchSpanProcessor( | ||
| 73 | new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT }) | ||
| 74 | ) | ||
| 75 | ) | ||
| 76 | |||
| 77 | tracerProvider.register() | ||
| 78 | } | ||
| 79 | |||
| 80 | async function wrapWithSpanAndContext <T> (spanName: string, cb: () => Promise<T>) { | ||
| 81 | const span = tracer.startSpan(spanName) | ||
| 82 | const activeContext = trace.setSpan(context.active(), span) | ||
| 83 | |||
| 84 | const result = await context.with(activeContext, () => cb()) | ||
| 85 | span.end() | ||
| 86 | |||
| 87 | return result | ||
| 88 | } | ||
| 89 | |||
| 90 | export { | ||
| 91 | registerOpentelemetryTracing, | ||
| 92 | tracer, | ||
| 93 | wrapWithSpanAndContext | ||
| 94 | } | ||
diff --git a/server/lib/paths.ts b/server/lib/paths.ts deleted file mode 100644 index db1cdede2..000000000 --- a/server/lib/paths.ts +++ /dev/null | |||
| @@ -1,92 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants' | ||
| 4 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
| 5 | import { removeFragmentedMP4Ext } from '@shared/core-utils' | ||
| 6 | import { buildUUID } from '@shared/extra-utils' | ||
| 7 | import { isVideoInPrivateDirectory } from './video-privacy' | ||
| 8 | |||
| 9 | // ################## Video file name ################## | ||
| 10 | |||
| 11 | function generateWebVideoFilename (resolution: number, extname: string) { | ||
| 12 | return buildUUID() + '-' + resolution + extname | ||
| 13 | } | ||
| 14 | |||
| 15 | function generateHLSVideoFilename (resolution: number) { | ||
| 16 | return `${buildUUID()}-${resolution}-fragmented.mp4` | ||
| 17 | } | ||
| 18 | |||
| 19 | // ################## Streaming playlist ################## | ||
| 20 | |||
| 21 | function getLiveDirectory (video: MVideo) { | ||
| 22 | return getHLSDirectory(video) | ||
| 23 | } | ||
| 24 | |||
| 25 | function getLiveReplayBaseDirectory (video: MVideo) { | ||
| 26 | return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) | ||
| 27 | } | ||
| 28 | |||
| 29 | function getHLSDirectory (video: MVideo) { | ||
| 30 | if (isVideoInPrivateDirectory(video.privacy)) { | ||
| 31 | return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid) | ||
| 32 | } | ||
| 33 | |||
| 34 | return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid) | ||
| 35 | } | ||
| 36 | |||
| 37 | function getHLSRedundancyDirectory (video: MVideoUUID) { | ||
| 38 | return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) | ||
| 39 | } | ||
| 40 | |||
| 41 | function getHlsResolutionPlaylistFilename (videoFilename: string) { | ||
| 42 | // Video file name already contain resolution | ||
| 43 | return removeFragmentedMP4Ext(videoFilename) + '.m3u8' | ||
| 44 | } | ||
| 45 | |||
| 46 | function generateHLSMasterPlaylistFilename (isLive = false) { | ||
| 47 | if (isLive) return 'master.m3u8' | ||
| 48 | |||
| 49 | return buildUUID() + '-master.m3u8' | ||
| 50 | } | ||
| 51 | |||
| 52 | function generateHlsSha256SegmentsFilename (isLive = false) { | ||
| 53 | if (isLive) return 'segments-sha256.json' | ||
| 54 | |||
| 55 | return buildUUID() + '-segments-sha256.json' | ||
| 56 | } | ||
| 57 | |||
| 58 | // ################## Torrents ################## | ||
| 59 | |||
| 60 | function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { | ||
| 61 | const extension = '.torrent' | ||
| 62 | const uuid = buildUUID() | ||
| 63 | |||
| 64 | if (isStreamingPlaylist(videoOrPlaylist)) { | ||
| 65 | return `${uuid}-${resolution}-${videoOrPlaylist.getStringType()}${extension}` | ||
| 66 | } | ||
| 67 | |||
| 68 | return uuid + '-' + resolution + extension | ||
| 69 | } | ||
| 70 | |||
| 71 | function getFSTorrentFilePath (videoFile: MVideoFile) { | ||
| 72 | return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) | ||
| 73 | } | ||
| 74 | |||
| 75 | // --------------------------------------------------------------------------- | ||
| 76 | |||
| 77 | export { | ||
| 78 | generateHLSVideoFilename, | ||
| 79 | generateWebVideoFilename, | ||
| 80 | |||
| 81 | generateTorrentFileName, | ||
| 82 | getFSTorrentFilePath, | ||
| 83 | |||
| 84 | getHLSDirectory, | ||
| 85 | getLiveDirectory, | ||
| 86 | getLiveReplayBaseDirectory, | ||
| 87 | getHLSRedundancyDirectory, | ||
| 88 | |||
| 89 | generateHLSMasterPlaylistFilename, | ||
| 90 | generateHlsSha256SegmentsFilename, | ||
| 91 | getHlsResolutionPlaylistFilename | ||
| 92 | } | ||
diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts deleted file mode 100644 index 3e41a2def..000000000 --- a/server/lib/peertube-socket.ts +++ /dev/null | |||
| @@ -1,129 +0,0 @@ | |||
| 1 | import { Server as HTTPServer } from 'http' | ||
| 2 | import { Namespace, Server as SocketServer, Socket } from 'socket.io' | ||
| 3 | import { isIdValid } from '@server/helpers/custom-validators/misc' | ||
| 4 | import { Debounce } from '@server/helpers/debounce' | ||
| 5 | import { MVideo, MVideoImmutable } from '@server/types/models' | ||
| 6 | import { MRunner } from '@server/types/models/runners' | ||
| 7 | import { UserNotificationModelForApi } from '@server/types/models/user' | ||
| 8 | import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models' | ||
| 9 | import { logger } from '../helpers/logger' | ||
| 10 | import { authenticateRunnerSocket, authenticateSocket } from '../middlewares' | ||
| 11 | |||
| 12 | class PeerTubeSocket { | ||
| 13 | |||
| 14 | private static instance: PeerTubeSocket | ||
| 15 | |||
| 16 | private userNotificationSockets: { [ userId: number ]: Socket[] } = {} | ||
| 17 | private liveVideosNamespace: Namespace | ||
| 18 | private readonly runnerSockets = new Set<Socket>() | ||
| 19 | |||
| 20 | private constructor () {} | ||
| 21 | |||
| 22 | init (server: HTTPServer) { | ||
| 23 | const io = new SocketServer(server) | ||
| 24 | |||
| 25 | io.of('/user-notifications') | ||
| 26 | .use(authenticateSocket) | ||
| 27 | .on('connection', socket => { | ||
| 28 | const userId = socket.handshake.auth.user.id | ||
| 29 | |||
| 30 | logger.debug('User %d connected to the notification system.', userId) | ||
| 31 | |||
| 32 | if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = [] | ||
| 33 | |||
| 34 | this.userNotificationSockets[userId].push(socket) | ||
| 35 | |||
| 36 | socket.on('disconnect', () => { | ||
| 37 | logger.debug('User %d disconnected from SocketIO notifications.', userId) | ||
| 38 | |||
| 39 | this.userNotificationSockets[userId] = this.userNotificationSockets[userId].filter(s => s !== socket) | ||
| 40 | }) | ||
| 41 | }) | ||
| 42 | |||
| 43 | this.liveVideosNamespace = io.of('/live-videos') | ||
| 44 | .on('connection', socket => { | ||
| 45 | socket.on('subscribe', ({ videoId }) => { | ||
| 46 | if (!isIdValid(videoId)) return | ||
| 47 | |||
| 48 | /* eslint-disable @typescript-eslint/no-floating-promises */ | ||
| 49 | socket.join(videoId) | ||
| 50 | }) | ||
| 51 | |||
| 52 | socket.on('unsubscribe', ({ videoId }) => { | ||
| 53 | if (!isIdValid(videoId)) return | ||
| 54 | |||
| 55 | /* eslint-disable @typescript-eslint/no-floating-promises */ | ||
| 56 | socket.leave(videoId) | ||
| 57 | }) | ||
| 58 | }) | ||
| 59 | |||
| 60 | io.of('/runners') | ||
| 61 | .use(authenticateRunnerSocket) | ||
| 62 | .on('connection', socket => { | ||
| 63 | const runner: MRunner = socket.handshake.auth.runner | ||
| 64 | |||
| 65 | logger.debug(`New runner "${runner.name}" connected to the notification system.`) | ||
| 66 | |||
| 67 | this.runnerSockets.add(socket) | ||
| 68 | |||
| 69 | socket.on('disconnect', () => { | ||
| 70 | logger.debug(`Runner "${runner.name}" disconnected from the notification system.`) | ||
| 71 | |||
| 72 | this.runnerSockets.delete(socket) | ||
| 73 | }) | ||
| 74 | }) | ||
| 75 | } | ||
| 76 | |||
| 77 | sendNotification (userId: number, notification: UserNotificationModelForApi) { | ||
| 78 | const sockets = this.userNotificationSockets[userId] | ||
| 79 | if (!sockets) return | ||
| 80 | |||
| 81 | logger.debug('Sending user notification to user %d.', userId) | ||
| 82 | |||
| 83 | const notificationMessage = notification.toFormattedJSON() | ||
| 84 | for (const socket of sockets) { | ||
| 85 | socket.emit('new-notification', notificationMessage) | ||
| 86 | } | ||
| 87 | } | ||
| 88 | |||
| 89 | sendVideoLiveNewState (video: MVideo) { | ||
| 90 | const data: LiveVideoEventPayload = { state: video.state } | ||
| 91 | const type: LiveVideoEventType = 'state-change' | ||
| 92 | |||
| 93 | logger.debug('Sending video live new state notification of %s.', video.url, { state: video.state }) | ||
| 94 | |||
| 95 | this.liveVideosNamespace | ||
| 96 | .in(video.id) | ||
| 97 | .emit(type, data) | ||
| 98 | } | ||
| 99 | |||
| 100 | sendVideoViewsUpdate (video: MVideoImmutable, numViewers: number) { | ||
| 101 | const data: LiveVideoEventPayload = { viewers: numViewers } | ||
| 102 | const type: LiveVideoEventType = 'views-change' | ||
| 103 | |||
| 104 | logger.debug('Sending video live views update notification of %s.', video.url, { viewers: numViewers }) | ||
| 105 | |||
| 106 | this.liveVideosNamespace | ||
| 107 | .in(video.id) | ||
| 108 | .emit(type, data) | ||
| 109 | } | ||
| 110 | |||
| 111 | @Debounce({ timeoutMS: 1000 }) | ||
| 112 | sendAvailableJobsPingToRunners () { | ||
| 113 | logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`) | ||
| 114 | |||
| 115 | for (const runners of this.runnerSockets) { | ||
| 116 | runners.emit('available-jobs') | ||
| 117 | } | ||
| 118 | } | ||
| 119 | |||
| 120 | static get Instance () { | ||
| 121 | return this.instance || (this.instance = new this()) | ||
| 122 | } | ||
| 123 | } | ||
| 124 | |||
| 125 | // --------------------------------------------------------------------------- | ||
| 126 | |||
| 127 | export { | ||
| 128 | PeerTubeSocket | ||
| 129 | } | ||
diff --git a/server/lib/plugins/hooks.ts b/server/lib/plugins/hooks.ts deleted file mode 100644 index 694527c12..000000000 --- a/server/lib/plugins/hooks.ts +++ /dev/null | |||
| @@ -1,35 +0,0 @@ | |||
| 1 | import Bluebird from 'bluebird' | ||
| 2 | import { ServerActionHookName, ServerFilterHookName } from '../../../shared/models' | ||
| 3 | import { logger } from '../../helpers/logger' | ||
| 4 | import { PluginManager } from './plugin-manager' | ||
| 5 | |||
| 6 | type PromiseFunction <U, T> = (params: U) => Promise<T> | Bluebird<T> | ||
| 7 | type RawFunction <U, T> = (params: U) => T | ||
| 8 | |||
| 9 | // Helpers to run hooks | ||
| 10 | const Hooks = { | ||
| 11 | wrapObject: <T, U extends ServerFilterHookName>(result: T, hookName: U, context?: any) => { | ||
| 12 | return PluginManager.Instance.runHook(hookName, result, context) | ||
| 13 | }, | ||
| 14 | |||
| 15 | wrapPromiseFun: async <U, T, V extends ServerFilterHookName>(fun: PromiseFunction<U, T>, params: U, hookName: V) => { | ||
| 16 | const result = await fun(params) | ||
| 17 | |||
| 18 | return PluginManager.Instance.runHook(hookName, result, params) | ||
| 19 | }, | ||
| 20 | |||
| 21 | wrapFun: async <U, T, V extends ServerFilterHookName>(fun: RawFunction<U, T>, params: U, hookName: V) => { | ||
| 22 | const result = fun(params) | ||
| 23 | |||
| 24 | return PluginManager.Instance.runHook(hookName, result, params) | ||
| 25 | }, | ||
| 26 | |||
| 27 | runAction: <T, U extends ServerActionHookName>(hookName: U, params?: T) => { | ||
| 28 | PluginManager.Instance.runHook(hookName, undefined, params) | ||
| 29 | .catch(err => logger.error('Fatal hook error.', { err })) | ||
| 30 | } | ||
| 31 | } | ||
| 32 | |||
| 33 | export { | ||
| 34 | Hooks | ||
| 35 | } | ||
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts deleted file mode 100644 index b4e3eece4..000000000 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ /dev/null | |||
| @@ -1,262 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { Server } from 'http' | ||
| 3 | import { join } from 'path' | ||
| 4 | import { buildLogger } from '@server/helpers/logger' | ||
| 5 | import { CONFIG } from '@server/initializers/config' | ||
| 6 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 7 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 8 | import { AccountModel } from '@server/models/account/account' | ||
| 9 | import { AccountBlocklistModel } from '@server/models/account/account-blocklist' | ||
| 10 | import { getServerActor } from '@server/models/application/application' | ||
| 11 | import { ServerModel } from '@server/models/server/server' | ||
| 12 | import { ServerBlocklistModel } from '@server/models/server/server-blocklist' | ||
| 13 | import { UserModel } from '@server/models/user/user' | ||
| 14 | import { VideoModel } from '@server/models/video/video' | ||
| 15 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' | ||
| 16 | import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models' | ||
| 17 | import { PeerTubeHelpers } from '@server/types/plugins' | ||
| 18 | import { ffprobePromise } from '@shared/ffmpeg' | ||
| 19 | import { VideoBlacklistCreate, VideoStorage } from '@shared/models' | ||
| 20 | import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' | ||
| 21 | import { PeerTubeSocket } from '../peertube-socket' | ||
| 22 | import { ServerConfigManager } from '../server-config-manager' | ||
| 23 | import { blacklistVideo, unblacklistVideo } from '../video-blacklist' | ||
| 24 | import { VideoPathManager } from '../video-path-manager' | ||
| 25 | |||
| 26 | function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers { | ||
| 27 | const logger = buildPluginLogger(npmName) | ||
| 28 | |||
| 29 | const database = buildDatabaseHelpers() | ||
| 30 | const videos = buildVideosHelpers() | ||
| 31 | |||
| 32 | const config = buildConfigHelpers() | ||
| 33 | |||
| 34 | const server = buildServerHelpers(httpServer) | ||
| 35 | |||
| 36 | const moderation = buildModerationHelpers() | ||
| 37 | |||
| 38 | const plugin = buildPluginRelatedHelpers(pluginModel, npmName) | ||
| 39 | |||
| 40 | const socket = buildSocketHelpers() | ||
| 41 | |||
| 42 | const user = buildUserHelpers() | ||
| 43 | |||
| 44 | return { | ||
| 45 | logger, | ||
| 46 | database, | ||
| 47 | videos, | ||
| 48 | config, | ||
| 49 | moderation, | ||
| 50 | plugin, | ||
| 51 | server, | ||
| 52 | socket, | ||
| 53 | user | ||
| 54 | } | ||
| 55 | } | ||
| 56 | |||
| 57 | export { | ||
| 58 | buildPluginHelpers | ||
| 59 | } | ||
| 60 | |||
| 61 | // --------------------------------------------------------------------------- | ||
| 62 | |||
| 63 | function buildPluginLogger (npmName: string) { | ||
| 64 | return buildLogger(npmName) | ||
| 65 | } | ||
| 66 | |||
| 67 | function buildDatabaseHelpers () { | ||
| 68 | return { | ||
| 69 | query: sequelizeTypescript.query.bind(sequelizeTypescript) | ||
| 70 | } | ||
| 71 | } | ||
| 72 | |||
| 73 | function buildServerHelpers (httpServer: Server) { | ||
| 74 | return { | ||
| 75 | getHTTPServer: () => httpServer, | ||
| 76 | |||
| 77 | getServerActor: () => getServerActor() | ||
| 78 | } | ||
| 79 | } | ||
| 80 | |||
| 81 | function buildVideosHelpers () { | ||
| 82 | return { | ||
| 83 | loadByUrl: (url: string) => { | ||
| 84 | return VideoModel.loadByUrl(url) | ||
| 85 | }, | ||
| 86 | |||
| 87 | loadByIdOrUUID: (id: number | string) => { | ||
| 88 | return VideoModel.load(id) | ||
| 89 | }, | ||
| 90 | |||
| 91 | removeVideo: (id: number) => { | ||
| 92 | return sequelizeTypescript.transaction(async t => { | ||
| 93 | const video = await VideoModel.loadFull(id, t) | ||
| 94 | |||
| 95 | await video.destroy({ transaction: t }) | ||
| 96 | }) | ||
| 97 | }, | ||
| 98 | |||
| 99 | ffprobe: (path: string) => { | ||
| 100 | return ffprobePromise(path) | ||
| 101 | }, | ||
| 102 | |||
| 103 | getFiles: async (id: number | string) => { | ||
| 104 | const video = await VideoModel.loadFull(id) | ||
| 105 | if (!video) return undefined | ||
| 106 | |||
| 107 | const webVideoFiles = (video.VideoFiles || []).map(f => ({ | ||
| 108 | path: f.storage === VideoStorage.FILE_SYSTEM | ||
| 109 | ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f) | ||
| 110 | : null, | ||
| 111 | url: f.getFileUrl(video), | ||
| 112 | |||
| 113 | resolution: f.resolution, | ||
| 114 | size: f.size, | ||
| 115 | fps: f.fps | ||
| 116 | })) | ||
| 117 | |||
| 118 | const hls = video.getHLSPlaylist() | ||
| 119 | |||
| 120 | const hlsVideoFiles = hls | ||
| 121 | ? (video.getHLSPlaylist().VideoFiles || []).map(f => { | ||
| 122 | return { | ||
| 123 | path: f.storage === VideoStorage.FILE_SYSTEM | ||
| 124 | ? VideoPathManager.Instance.getFSVideoFileOutputPath(hls, f) | ||
| 125 | : null, | ||
| 126 | url: f.getFileUrl(video), | ||
| 127 | resolution: f.resolution, | ||
| 128 | size: f.size, | ||
| 129 | fps: f.fps | ||
| 130 | } | ||
| 131 | }) | ||
| 132 | : [] | ||
| 133 | |||
| 134 | const thumbnails = video.Thumbnails.map(t => ({ | ||
| 135 | type: t.type, | ||
| 136 | url: t.getOriginFileUrl(video), | ||
| 137 | path: t.getPath() | ||
| 138 | })) | ||
| 139 | |||
| 140 | return { | ||
| 141 | webtorrent: { // TODO: remove in v7 | ||
| 142 | videoFiles: webVideoFiles | ||
| 143 | }, | ||
| 144 | |||
| 145 | webVideo: { | ||
| 146 | videoFiles: webVideoFiles | ||
| 147 | }, | ||
| 148 | |||
| 149 | hls: { | ||
| 150 | videoFiles: hlsVideoFiles | ||
| 151 | }, | ||
| 152 | |||
| 153 | thumbnails | ||
| 154 | } | ||
| 155 | } | ||
| 156 | } | ||
| 157 | } | ||
| 158 | |||
| 159 | function buildModerationHelpers () { | ||
| 160 | return { | ||
| 161 | blockServer: async (options: { byAccountId: number, hostToBlock: string }) => { | ||
| 162 | const serverToBlock = await ServerModel.loadOrCreateByHost(options.hostToBlock) | ||
| 163 | |||
| 164 | await addServerInBlocklist(options.byAccountId, serverToBlock.id) | ||
| 165 | }, | ||
| 166 | |||
| 167 | unblockServer: async (options: { byAccountId: number, hostToUnblock: string }) => { | ||
| 168 | const serverBlock = await ServerBlocklistModel.loadByAccountAndHost(options.byAccountId, options.hostToUnblock) | ||
| 169 | if (!serverBlock) return | ||
| 170 | |||
| 171 | await removeServerFromBlocklist(serverBlock) | ||
| 172 | }, | ||
| 173 | |||
| 174 | blockAccount: async (options: { byAccountId: number, handleToBlock: string }) => { | ||
| 175 | const accountToBlock = await AccountModel.loadByNameWithHost(options.handleToBlock) | ||
| 176 | if (!accountToBlock) return | ||
| 177 | |||
| 178 | await addAccountInBlocklist(options.byAccountId, accountToBlock.id) | ||
| 179 | }, | ||
| 180 | |||
| 181 | unblockAccount: async (options: { byAccountId: number, handleToUnblock: string }) => { | ||
| 182 | const targetAccount = await AccountModel.loadByNameWithHost(options.handleToUnblock) | ||
| 183 | if (!targetAccount) return | ||
| 184 | |||
| 185 | const accountBlock = await AccountBlocklistModel.loadByAccountAndTarget(options.byAccountId, targetAccount.id) | ||
| 186 | if (!accountBlock) return | ||
| 187 | |||
| 188 | await removeAccountFromBlocklist(accountBlock) | ||
| 189 | }, | ||
| 190 | |||
| 191 | blacklistVideo: async (options: { videoIdOrUUID: number | string, createOptions: VideoBlacklistCreate }) => { | ||
| 192 | const video = await VideoModel.loadFull(options.videoIdOrUUID) | ||
| 193 | if (!video) return | ||
| 194 | |||
| 195 | await blacklistVideo(video, options.createOptions) | ||
| 196 | }, | ||
| 197 | |||
| 198 | unblacklistVideo: async (options: { videoIdOrUUID: number | string }) => { | ||
| 199 | const video = await VideoModel.loadFull(options.videoIdOrUUID) | ||
| 200 | if (!video) return | ||
| 201 | |||
| 202 | const videoBlacklist = await VideoBlacklistModel.loadByVideoId(video.id) | ||
| 203 | if (!videoBlacklist) return | ||
| 204 | |||
| 205 | await unblacklistVideo(videoBlacklist, video) | ||
| 206 | } | ||
| 207 | } | ||
| 208 | } | ||
| 209 | |||
| 210 | function buildConfigHelpers () { | ||
| 211 | return { | ||
| 212 | getWebserverUrl () { | ||
| 213 | return WEBSERVER.URL | ||
| 214 | }, | ||
| 215 | |||
| 216 | getServerListeningConfig () { | ||
| 217 | return { hostname: CONFIG.LISTEN.HOSTNAME, port: CONFIG.LISTEN.PORT } | ||
| 218 | }, | ||
| 219 | |||
| 220 | getServerConfig () { | ||
| 221 | return ServerConfigManager.Instance.getServerConfig() | ||
| 222 | } | ||
| 223 | } | ||
| 224 | } | ||
| 225 | |||
| 226 | function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) { | ||
| 227 | return { | ||
| 228 | getBaseStaticRoute: () => `/plugins/${plugin.name}/${plugin.version}/static/`, | ||
| 229 | |||
| 230 | getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, | ||
| 231 | |||
| 232 | getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`, | ||
| 233 | |||
| 234 | getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) | ||
| 235 | } | ||
| 236 | } | ||
| 237 | |||
| 238 | function buildSocketHelpers () { | ||
| 239 | return { | ||
| 240 | sendNotification: (userId: number, notification: UserNotificationModelForApi) => { | ||
| 241 | PeerTubeSocket.Instance.sendNotification(userId, notification) | ||
| 242 | }, | ||
| 243 | sendVideoLiveNewState: (video: MVideo) => { | ||
| 244 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | ||
| 245 | } | ||
| 246 | } | ||
| 247 | } | ||
| 248 | |||
| 249 | function buildUserHelpers () { | ||
| 250 | return { | ||
| 251 | loadById: (id: number) => { | ||
| 252 | return UserModel.loadByIdFull(id) | ||
| 253 | }, | ||
| 254 | |||
| 255 | getAuthUser: (res: express.Response) => { | ||
| 256 | const user = res.locals.oauth?.token?.User || res.locals.videoFileToken?.user | ||
| 257 | if (!user) return undefined | ||
| 258 | |||
| 259 | return UserModel.loadByIdFull(user.id) | ||
| 260 | } | ||
| 261 | } | ||
| 262 | } | ||
diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts deleted file mode 100644 index 119cee8e0..000000000 --- a/server/lib/plugins/plugin-index.ts +++ /dev/null | |||
| @@ -1,85 +0,0 @@ | |||
| 1 | import { sanitizeUrl } from '@server/helpers/core-utils' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { doJSONRequest } from '@server/helpers/requests' | ||
| 4 | import { CONFIG } from '@server/initializers/config' | ||
| 5 | import { PEERTUBE_VERSION } from '@server/initializers/constants' | ||
| 6 | import { PluginModel } from '@server/models/server/plugin' | ||
| 7 | import { | ||
| 8 | PeerTubePluginIndex, | ||
| 9 | PeertubePluginIndexList, | ||
| 10 | PeertubePluginLatestVersionRequest, | ||
| 11 | PeertubePluginLatestVersionResponse, | ||
| 12 | ResultList | ||
| 13 | } from '@shared/models' | ||
| 14 | import { PluginManager } from './plugin-manager' | ||
| 15 | |||
| 16 | async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { | ||
| 17 | const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options | ||
| 18 | |||
| 19 | const searchParams: PeertubePluginIndexList & Record<string, string | number> = { | ||
| 20 | start, | ||
| 21 | count, | ||
| 22 | sort, | ||
| 23 | pluginType, | ||
| 24 | search, | ||
| 25 | currentPeerTubeEngine: options.currentPeerTubeEngine || PEERTUBE_VERSION | ||
| 26 | } | ||
| 27 | |||
| 28 | const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' | ||
| 29 | |||
| 30 | try { | ||
| 31 | const { body } = await doJSONRequest<any>(uri, { searchParams }) | ||
| 32 | |||
| 33 | logger.debug('Got result from PeerTube index.', { body }) | ||
| 34 | |||
| 35 | addInstanceInformation(body) | ||
| 36 | |||
| 37 | return body as ResultList<PeerTubePluginIndex> | ||
| 38 | } catch (err) { | ||
| 39 | logger.error('Cannot list available plugins from index %s.', uri, { err }) | ||
| 40 | return undefined | ||
| 41 | } | ||
| 42 | } | ||
| 43 | |||
| 44 | function addInstanceInformation (result: ResultList<PeerTubePluginIndex>) { | ||
| 45 | for (const d of result.data) { | ||
| 46 | d.installed = PluginManager.Instance.isRegistered(d.npmName) | ||
| 47 | d.name = PluginModel.normalizePluginName(d.npmName) | ||
| 48 | } | ||
| 49 | |||
| 50 | return result | ||
| 51 | } | ||
| 52 | |||
| 53 | async function getLatestPluginsVersion (npmNames: string[]): Promise<PeertubePluginLatestVersionResponse> { | ||
| 54 | const bodyRequest: PeertubePluginLatestVersionRequest = { | ||
| 55 | npmNames, | ||
| 56 | currentPeerTubeEngine: PEERTUBE_VERSION | ||
| 57 | } | ||
| 58 | |||
| 59 | const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version' | ||
| 60 | |||
| 61 | const options = { | ||
| 62 | json: bodyRequest, | ||
| 63 | method: 'POST' as 'POST' | ||
| 64 | } | ||
| 65 | const { body } = await doJSONRequest<PeertubePluginLatestVersionResponse>(uri, options) | ||
| 66 | |||
| 67 | return body | ||
| 68 | } | ||
| 69 | |||
| 70 | async function getLatestPluginVersion (npmName: string) { | ||
| 71 | const results = await getLatestPluginsVersion([ npmName ]) | ||
| 72 | |||
| 73 | if (Array.isArray(results) === false || results.length !== 1) { | ||
| 74 | logger.warn('Cannot get latest supported plugin version of %s.', npmName) | ||
| 75 | return undefined | ||
| 76 | } | ||
| 77 | |||
| 78 | return results[0].latestVersion | ||
| 79 | } | ||
| 80 | |||
| 81 | export { | ||
| 82 | listAvailablePluginsFromIndex, | ||
| 83 | getLatestPluginVersion, | ||
| 84 | getLatestPluginsVersion | ||
| 85 | } | ||
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts deleted file mode 100644 index 88c5b60d7..000000000 --- a/server/lib/plugins/plugin-manager.ts +++ /dev/null | |||
| @@ -1,665 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { createReadStream, createWriteStream } from 'fs' | ||
| 3 | import { ensureDir, outputFile, readJSON } from 'fs-extra' | ||
| 4 | import { Server } from 'http' | ||
| 5 | import { basename, join } from 'path' | ||
| 6 | import { decachePlugin } from '@server/helpers/decache' | ||
| 7 | import { ApplicationModel } from '@server/models/application/application' | ||
| 8 | import { MOAuthTokenUser, MUser } from '@server/types/models' | ||
| 9 | import { getCompleteLocale } from '@shared/core-utils' | ||
| 10 | import { | ||
| 11 | ClientScriptJSON, | ||
| 12 | PluginPackageJSON, | ||
| 13 | PluginTranslation, | ||
| 14 | PluginTranslationPathsJSON, | ||
| 15 | RegisterServerHookOptions | ||
| 16 | } from '@shared/models' | ||
| 17 | import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' | ||
| 18 | import { PluginType } from '../../../shared/models/plugins/plugin.type' | ||
| 19 | import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' | ||
| 20 | import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins' | ||
| 21 | import { logger } from '../../helpers/logger' | ||
| 22 | import { CONFIG } from '../../initializers/config' | ||
| 23 | import { PLUGIN_GLOBAL_CSS_PATH } from '../../initializers/constants' | ||
| 24 | import { PluginModel } from '../../models/server/plugin' | ||
| 25 | import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPassOptions, RegisterServerOptions } from '../../types/plugins' | ||
| 26 | import { ClientHtml } from '../client-html' | ||
| 27 | import { RegisterHelpers } from './register-helpers' | ||
| 28 | import { installNpmPlugin, installNpmPluginFromDisk, rebuildNativePlugins, removeNpmPlugin } from './yarn' | ||
| 29 | |||
| 30 | export interface RegisteredPlugin { | ||
| 31 | npmName: string | ||
| 32 | name: string | ||
| 33 | version: string | ||
| 34 | description: string | ||
| 35 | peertubeEngine: string | ||
| 36 | |||
| 37 | type: PluginType | ||
| 38 | |||
| 39 | path: string | ||
| 40 | |||
| 41 | staticDirs: { [name: string]: string } | ||
| 42 | clientScripts: { [name: string]: ClientScriptJSON } | ||
| 43 | |||
| 44 | css: string[] | ||
| 45 | |||
| 46 | // Only if this is a plugin | ||
| 47 | registerHelpers?: RegisterHelpers | ||
| 48 | unregister?: Function | ||
| 49 | } | ||
| 50 | |||
| 51 | export interface HookInformationValue { | ||
| 52 | npmName: string | ||
| 53 | pluginName: string | ||
| 54 | handler: Function | ||
| 55 | priority: number | ||
| 56 | } | ||
| 57 | |||
| 58 | type PluginLocalesTranslations = { | ||
| 59 | [locale: string]: PluginTranslation | ||
| 60 | } | ||
| 61 | |||
| 62 | export class PluginManager implements ServerHook { | ||
| 63 | |||
| 64 | private static instance: PluginManager | ||
| 65 | |||
| 66 | private registeredPlugins: { [name: string]: RegisteredPlugin } = {} | ||
| 67 | |||
| 68 | private hooks: { [name: string]: HookInformationValue[] } = {} | ||
| 69 | private translations: PluginLocalesTranslations = {} | ||
| 70 | |||
| 71 | private server: Server | ||
| 72 | |||
| 73 | private constructor () { | ||
| 74 | } | ||
| 75 | |||
| 76 | init (server: Server) { | ||
| 77 | this.server = server | ||
| 78 | } | ||
| 79 | |||
| 80 | registerWebSocketRouter () { | ||
| 81 | this.server.on('upgrade', (request, socket, head) => { | ||
| 82 | // Check if it's a plugin websocket connection | ||
| 83 | // No need to destroy the stream when we abort the request | ||
| 84 | // Other handlers in PeerTube will catch this upgrade event too (socket.io, tracker etc) | ||
| 85 | |||
| 86 | const url = request.url | ||
| 87 | |||
| 88 | const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`) | ||
| 89 | if (!matched) return | ||
| 90 | |||
| 91 | const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN) | ||
| 92 | const subRoute = matched[3] | ||
| 93 | |||
| 94 | const result = this.getRegisteredPluginOrTheme(npmName) | ||
| 95 | if (!result) return | ||
| 96 | |||
| 97 | const routes = result.registerHelpers.getWebSocketRoutes() | ||
| 98 | |||
| 99 | const wss = routes.find(r => r.route.startsWith(subRoute)) | ||
| 100 | if (!wss) return | ||
| 101 | |||
| 102 | try { | ||
| 103 | wss.handler(request, socket, head) | ||
| 104 | } catch (err) { | ||
| 105 | logger.error('Exception in plugin handler ' + npmName, { err }) | ||
| 106 | } | ||
| 107 | }) | ||
| 108 | } | ||
| 109 | |||
| 110 | // ###################### Getters ###################### | ||
| 111 | |||
| 112 | isRegistered (npmName: string) { | ||
| 113 | return !!this.getRegisteredPluginOrTheme(npmName) | ||
| 114 | } | ||
| 115 | |||
| 116 | getRegisteredPluginOrTheme (npmName: string) { | ||
| 117 | return this.registeredPlugins[npmName] | ||
| 118 | } | ||
| 119 | |||
| 120 | getRegisteredPluginByShortName (name: string) { | ||
| 121 | const npmName = PluginModel.buildNpmName(name, PluginType.PLUGIN) | ||
| 122 | const registered = this.getRegisteredPluginOrTheme(npmName) | ||
| 123 | |||
| 124 | if (!registered || registered.type !== PluginType.PLUGIN) return undefined | ||
| 125 | |||
| 126 | return registered | ||
| 127 | } | ||
| 128 | |||
| 129 | getRegisteredThemeByShortName (name: string) { | ||
| 130 | const npmName = PluginModel.buildNpmName(name, PluginType.THEME) | ||
| 131 | const registered = this.getRegisteredPluginOrTheme(npmName) | ||
| 132 | |||
| 133 | if (!registered || registered.type !== PluginType.THEME) return undefined | ||
| 134 | |||
| 135 | return registered | ||
| 136 | } | ||
| 137 | |||
| 138 | getRegisteredPlugins () { | ||
| 139 | return this.getRegisteredPluginsOrThemes(PluginType.PLUGIN) | ||
| 140 | } | ||
| 141 | |||
| 142 | getRegisteredThemes () { | ||
| 143 | return this.getRegisteredPluginsOrThemes(PluginType.THEME) | ||
| 144 | } | ||
| 145 | |||
| 146 | getIdAndPassAuths () { | ||
| 147 | return this.getRegisteredPlugins() | ||
| 148 | .map(p => ({ | ||
| 149 | npmName: p.npmName, | ||
| 150 | name: p.name, | ||
| 151 | version: p.version, | ||
| 152 | idAndPassAuths: p.registerHelpers.getIdAndPassAuths() | ||
| 153 | })) | ||
| 154 | .filter(v => v.idAndPassAuths.length !== 0) | ||
| 155 | } | ||
| 156 | |||
| 157 | getExternalAuths () { | ||
| 158 | return this.getRegisteredPlugins() | ||
| 159 | .map(p => ({ | ||
| 160 | npmName: p.npmName, | ||
| 161 | name: p.name, | ||
| 162 | version: p.version, | ||
| 163 | externalAuths: p.registerHelpers.getExternalAuths() | ||
| 164 | })) | ||
| 165 | .filter(v => v.externalAuths.length !== 0) | ||
| 166 | } | ||
| 167 | |||
| 168 | getRegisteredSettings (npmName: string) { | ||
| 169 | const result = this.getRegisteredPluginOrTheme(npmName) | ||
| 170 | if (!result || result.type !== PluginType.PLUGIN) return [] | ||
| 171 | |||
| 172 | return result.registerHelpers.getSettings() | ||
| 173 | } | ||
| 174 | |||
| 175 | getRouter (npmName: string) { | ||
| 176 | const result = this.getRegisteredPluginOrTheme(npmName) | ||
| 177 | if (!result || result.type !== PluginType.PLUGIN) return null | ||
| 178 | |||
| 179 | return result.registerHelpers.getRouter() | ||
| 180 | } | ||
| 181 | |||
| 182 | getTranslations (locale: string) { | ||
| 183 | return this.translations[locale] || {} | ||
| 184 | } | ||
| 185 | |||
| 186 | async isTokenValid (token: MOAuthTokenUser, type: 'access' | 'refresh') { | ||
| 187 | const auth = this.getAuth(token.User.pluginAuth, token.authName) | ||
| 188 | if (!auth) return true | ||
| 189 | |||
| 190 | if (auth.hookTokenValidity) { | ||
| 191 | try { | ||
| 192 | const { valid } = await auth.hookTokenValidity({ token, type }) | ||
| 193 | |||
| 194 | if (valid === false) { | ||
| 195 | logger.info('Rejecting %s token validity from auth %s of plugin %s', type, token.authName, token.User.pluginAuth) | ||
| 196 | } | ||
| 197 | |||
| 198 | return valid | ||
| 199 | } catch (err) { | ||
| 200 | logger.warn('Cannot run check token validity from auth %s of plugin %s.', token.authName, token.User.pluginAuth, { err }) | ||
| 201 | return true | ||
| 202 | } | ||
| 203 | } | ||
| 204 | |||
| 205 | return true | ||
| 206 | } | ||
| 207 | |||
| 208 | // ###################### External events ###################### | ||
| 209 | |||
| 210 | async onLogout (npmName: string, authName: string, user: MUser, req: express.Request) { | ||
| 211 | const auth = this.getAuth(npmName, authName) | ||
| 212 | |||
| 213 | if (auth?.onLogout) { | ||
| 214 | logger.info('Running onLogout function from auth %s of plugin %s', authName, npmName) | ||
| 215 | |||
| 216 | try { | ||
| 217 | // Force await, in case or onLogout returns a promise | ||
| 218 | const result = await auth.onLogout(user, req) | ||
| 219 | |||
| 220 | return typeof result === 'string' | ||
| 221 | ? result | ||
| 222 | : undefined | ||
| 223 | } catch (err) { | ||
| 224 | logger.warn('Cannot run onLogout function from auth %s of plugin %s.', authName, npmName, { err }) | ||
| 225 | } | ||
| 226 | } | ||
| 227 | |||
| 228 | return undefined | ||
| 229 | } | ||
| 230 | |||
| 231 | async onSettingsChanged (name: string, settings: any) { | ||
| 232 | const registered = this.getRegisteredPluginByShortName(name) | ||
| 233 | if (!registered) { | ||
| 234 | logger.error('Cannot find plugin %s to call on settings changed.', name) | ||
| 235 | } | ||
| 236 | |||
| 237 | for (const cb of registered.registerHelpers.getOnSettingsChangedCallbacks()) { | ||
| 238 | try { | ||
| 239 | await cb(settings) | ||
| 240 | } catch (err) { | ||
| 241 | logger.error('Cannot run on settings changed callback for %s.', registered.npmName, { err }) | ||
| 242 | } | ||
| 243 | } | ||
| 244 | } | ||
| 245 | |||
| 246 | // ###################### Hooks ###################### | ||
| 247 | |||
| 248 | async runHook<T> (hookName: ServerHookName, result?: T, params?: any): Promise<T> { | ||
| 249 | if (!this.hooks[hookName]) return Promise.resolve(result) | ||
| 250 | |||
| 251 | const hookType = getHookType(hookName) | ||
| 252 | |||
| 253 | for (const hook of this.hooks[hookName]) { | ||
| 254 | logger.debug('Running hook %s of plugin %s.', hookName, hook.npmName) | ||
| 255 | |||
| 256 | result = await internalRunHook({ | ||
| 257 | handler: hook.handler, | ||
| 258 | hookType, | ||
| 259 | result, | ||
| 260 | params, | ||
| 261 | onError: err => { logger.error('Cannot run hook %s of plugin %s.', hookName, hook.pluginName, { err }) } | ||
| 262 | }) | ||
| 263 | } | ||
| 264 | |||
| 265 | return result | ||
| 266 | } | ||
| 267 | |||
| 268 | // ###################### Registration ###################### | ||
| 269 | |||
| 270 | async registerPluginsAndThemes () { | ||
| 271 | await this.resetCSSGlobalFile() | ||
| 272 | |||
| 273 | const plugins = await PluginModel.listEnabledPluginsAndThemes() | ||
| 274 | |||
| 275 | for (const plugin of plugins) { | ||
| 276 | try { | ||
| 277 | await this.registerPluginOrTheme(plugin) | ||
| 278 | } catch (err) { | ||
| 279 | // Try to unregister the plugin | ||
| 280 | try { | ||
| 281 | await this.unregister(PluginModel.buildNpmName(plugin.name, plugin.type)) | ||
| 282 | } catch { | ||
| 283 | // we don't care if we cannot unregister it | ||
| 284 | } | ||
| 285 | |||
| 286 | logger.error('Cannot register plugin %s, skipping.', plugin.name, { err }) | ||
| 287 | } | ||
| 288 | } | ||
| 289 | |||
| 290 | this.sortHooksByPriority() | ||
| 291 | } | ||
| 292 | |||
| 293 | // Don't need the plugin type since themes cannot register server code | ||
| 294 | async unregister (npmName: string) { | ||
| 295 | logger.info('Unregister plugin %s.', npmName) | ||
| 296 | |||
| 297 | const plugin = this.getRegisteredPluginOrTheme(npmName) | ||
| 298 | |||
| 299 | if (!plugin) { | ||
| 300 | throw new Error(`Unknown plugin ${npmName} to unregister`) | ||
| 301 | } | ||
| 302 | |||
| 303 | delete this.registeredPlugins[plugin.npmName] | ||
| 304 | |||
| 305 | this.deleteTranslations(plugin.npmName) | ||
| 306 | |||
| 307 | if (plugin.type === PluginType.PLUGIN) { | ||
| 308 | await plugin.unregister() | ||
| 309 | |||
| 310 | // Remove hooks of this plugin | ||
| 311 | for (const key of Object.keys(this.hooks)) { | ||
| 312 | this.hooks[key] = this.hooks[key].filter(h => h.npmName !== npmName) | ||
| 313 | } | ||
| 314 | |||
| 315 | const store = plugin.registerHelpers | ||
| 316 | store.reinitVideoConstants(plugin.npmName) | ||
| 317 | store.reinitTranscodingProfilesAndEncoders(plugin.npmName) | ||
| 318 | |||
| 319 | logger.info('Regenerating registered plugin CSS to global file.') | ||
| 320 | await this.regeneratePluginGlobalCSS() | ||
| 321 | } | ||
| 322 | |||
| 323 | ClientHtml.invalidCache() | ||
| 324 | } | ||
| 325 | |||
| 326 | // ###################### Installation ###################### | ||
| 327 | |||
| 328 | async install (options: { | ||
| 329 | toInstall: string | ||
| 330 | version?: string | ||
| 331 | fromDisk?: boolean // default false | ||
| 332 | register?: boolean // default true | ||
| 333 | }) { | ||
| 334 | const { toInstall, version, fromDisk = false, register = true } = options | ||
| 335 | |||
| 336 | let plugin: PluginModel | ||
| 337 | let npmName: string | ||
| 338 | |||
| 339 | logger.info('Installing plugin %s.', toInstall) | ||
| 340 | |||
| 341 | try { | ||
| 342 | fromDisk | ||
| 343 | ? await installNpmPluginFromDisk(toInstall) | ||
| 344 | : await installNpmPlugin(toInstall, version) | ||
| 345 | |||
| 346 | npmName = fromDisk ? basename(toInstall) : toInstall | ||
| 347 | const pluginType = PluginModel.getTypeFromNpmName(npmName) | ||
| 348 | const pluginName = PluginModel.normalizePluginName(npmName) | ||
| 349 | |||
| 350 | const packageJSON = await this.getPackageJSON(pluginName, pluginType) | ||
| 351 | |||
| 352 | this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, pluginType); | ||
| 353 | |||
| 354 | [ plugin ] = await PluginModel.upsert({ | ||
| 355 | name: pluginName, | ||
| 356 | description: packageJSON.description, | ||
| 357 | homepage: packageJSON.homepage, | ||
| 358 | type: pluginType, | ||
| 359 | version: packageJSON.version, | ||
| 360 | enabled: true, | ||
| 361 | uninstalled: false, | ||
| 362 | peertubeEngine: packageJSON.engine.peertube | ||
| 363 | }, { returning: true }) | ||
| 364 | |||
| 365 | logger.info('Successful installation of plugin %s.', toInstall) | ||
| 366 | |||
| 367 | if (register) { | ||
| 368 | await this.registerPluginOrTheme(plugin) | ||
| 369 | } | ||
| 370 | } catch (rootErr) { | ||
| 371 | logger.error('Cannot install plugin %s, removing it...', toInstall, { err: rootErr }) | ||
| 372 | |||
| 373 | if (npmName) { | ||
| 374 | try { | ||
| 375 | await this.uninstall({ npmName }) | ||
| 376 | } catch (err) { | ||
| 377 | logger.error('Cannot uninstall plugin %s after failed installation.', toInstall, { err }) | ||
| 378 | |||
| 379 | try { | ||
| 380 | await removeNpmPlugin(npmName) | ||
| 381 | } catch (err) { | ||
| 382 | logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) | ||
| 383 | } | ||
| 384 | } | ||
| 385 | } | ||
| 386 | |||
| 387 | throw rootErr | ||
| 388 | } | ||
| 389 | |||
| 390 | return plugin | ||
| 391 | } | ||
| 392 | |||
| 393 | async update (toUpdate: string, fromDisk = false) { | ||
| 394 | const npmName = fromDisk ? basename(toUpdate) : toUpdate | ||
| 395 | |||
| 396 | logger.info('Updating plugin %s.', npmName) | ||
| 397 | |||
| 398 | // Use the latest version from DB, to not upgrade to a version that does not support our PeerTube version | ||
| 399 | let version: string | ||
| 400 | if (!fromDisk) { | ||
| 401 | const plugin = await PluginModel.loadByNpmName(toUpdate) | ||
| 402 | version = plugin.latestVersion | ||
| 403 | } | ||
| 404 | |||
| 405 | // Unregister old hooks | ||
| 406 | await this.unregister(npmName) | ||
| 407 | |||
| 408 | return this.install({ toInstall: toUpdate, version, fromDisk }) | ||
| 409 | } | ||
| 410 | |||
| 411 | async uninstall (options: { | ||
| 412 | npmName: string | ||
| 413 | unregister?: boolean // default true | ||
| 414 | }) { | ||
| 415 | const { npmName, unregister = true } = options | ||
| 416 | |||
| 417 | logger.info('Uninstalling plugin %s.', npmName) | ||
| 418 | |||
| 419 | if (unregister) { | ||
| 420 | try { | ||
| 421 | await this.unregister(npmName) | ||
| 422 | } catch (err) { | ||
| 423 | logger.warn('Cannot unregister plugin %s.', npmName, { err }) | ||
| 424 | } | ||
| 425 | } | ||
| 426 | |||
| 427 | const plugin = await PluginModel.loadByNpmName(npmName) | ||
| 428 | if (!plugin || plugin.uninstalled === true) { | ||
| 429 | logger.error('Cannot uninstall plugin %s: it does not exist or is already uninstalled.', npmName) | ||
| 430 | return | ||
| 431 | } | ||
| 432 | |||
| 433 | plugin.enabled = false | ||
| 434 | plugin.uninstalled = true | ||
| 435 | |||
| 436 | await plugin.save() | ||
| 437 | |||
| 438 | await removeNpmPlugin(npmName) | ||
| 439 | |||
| 440 | logger.info('Plugin %s uninstalled.', npmName) | ||
| 441 | } | ||
| 442 | |||
| 443 | async rebuildNativePluginsIfNeeded () { | ||
| 444 | if (!await ApplicationModel.nodeABIChanged()) return | ||
| 445 | |||
| 446 | return rebuildNativePlugins() | ||
| 447 | } | ||
| 448 | |||
| 449 | // ###################### Private register ###################### | ||
| 450 | |||
| 451 | private async registerPluginOrTheme (plugin: PluginModel) { | ||
| 452 | const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) | ||
| 453 | |||
| 454 | logger.info('Registering plugin or theme %s.', npmName) | ||
| 455 | |||
| 456 | const packageJSON = await this.getPackageJSON(plugin.name, plugin.type) | ||
| 457 | const pluginPath = this.getPluginPath(plugin.name, plugin.type) | ||
| 458 | |||
| 459 | this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type) | ||
| 460 | |||
| 461 | let library: PluginLibrary | ||
| 462 | let registerHelpers: RegisterHelpers | ||
| 463 | if (plugin.type === PluginType.PLUGIN) { | ||
| 464 | const result = await this.registerPlugin(plugin, pluginPath, packageJSON) | ||
| 465 | library = result.library | ||
| 466 | registerHelpers = result.registerStore | ||
| 467 | } | ||
| 468 | |||
| 469 | const clientScripts: { [id: string]: ClientScriptJSON } = {} | ||
| 470 | for (const c of packageJSON.clientScripts) { | ||
| 471 | clientScripts[c.script] = c | ||
| 472 | } | ||
| 473 | |||
| 474 | this.registeredPlugins[npmName] = { | ||
| 475 | npmName, | ||
| 476 | name: plugin.name, | ||
| 477 | type: plugin.type, | ||
| 478 | version: plugin.version, | ||
| 479 | description: plugin.description, | ||
| 480 | peertubeEngine: plugin.peertubeEngine, | ||
| 481 | path: pluginPath, | ||
| 482 | staticDirs: packageJSON.staticDirs, | ||
| 483 | clientScripts, | ||
| 484 | css: packageJSON.css, | ||
| 485 | registerHelpers: registerHelpers || undefined, | ||
| 486 | unregister: library ? library.unregister : undefined | ||
| 487 | } | ||
| 488 | |||
| 489 | await this.addTranslations(plugin, npmName, packageJSON.translations) | ||
| 490 | |||
| 491 | ClientHtml.invalidCache() | ||
| 492 | } | ||
| 493 | |||
| 494 | private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) { | ||
| 495 | const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) | ||
| 496 | |||
| 497 | // Delete cache if needed | ||
| 498 | const modulePath = join(pluginPath, packageJSON.library) | ||
| 499 | decachePlugin(modulePath) | ||
| 500 | const library: PluginLibrary = require(modulePath) | ||
| 501 | |||
| 502 | if (!isLibraryCodeValid(library)) { | ||
| 503 | throw new Error('Library code is not valid (miss register or unregister function)') | ||
| 504 | } | ||
| 505 | |||
| 506 | const { registerOptions, registerStore } = this.getRegisterHelpers(npmName, plugin) | ||
| 507 | |||
| 508 | await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath()) | ||
| 509 | |||
| 510 | await library.register(registerOptions) | ||
| 511 | |||
| 512 | logger.info('Add plugin %s CSS to global file.', npmName) | ||
| 513 | |||
| 514 | await this.addCSSToGlobalFile(pluginPath, packageJSON.css) | ||
| 515 | |||
| 516 | return { library, registerStore } | ||
| 517 | } | ||
| 518 | |||
| 519 | // ###################### Translations ###################### | ||
| 520 | |||
| 521 | private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) { | ||
| 522 | for (const locale of Object.keys(translationPaths)) { | ||
| 523 | const path = translationPaths[locale] | ||
| 524 | const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) | ||
| 525 | |||
| 526 | const completeLocale = getCompleteLocale(locale) | ||
| 527 | |||
| 528 | if (!this.translations[completeLocale]) this.translations[completeLocale] = {} | ||
| 529 | this.translations[completeLocale][npmName] = json | ||
| 530 | |||
| 531 | logger.info('Added locale %s of plugin %s.', completeLocale, npmName) | ||
| 532 | } | ||
| 533 | } | ||
| 534 | |||
| 535 | private deleteTranslations (npmName: string) { | ||
| 536 | for (const locale of Object.keys(this.translations)) { | ||
| 537 | delete this.translations[locale][npmName] | ||
| 538 | |||
| 539 | logger.info('Deleted locale %s of plugin %s.', locale, npmName) | ||
| 540 | } | ||
| 541 | } | ||
| 542 | |||
| 543 | // ###################### CSS ###################### | ||
| 544 | |||
| 545 | private resetCSSGlobalFile () { | ||
| 546 | return outputFile(PLUGIN_GLOBAL_CSS_PATH, '') | ||
| 547 | } | ||
| 548 | |||
| 549 | private async addCSSToGlobalFile (pluginPath: string, cssRelativePaths: string[]) { | ||
| 550 | for (const cssPath of cssRelativePaths) { | ||
| 551 | await this.concatFiles(join(pluginPath, cssPath), PLUGIN_GLOBAL_CSS_PATH) | ||
| 552 | } | ||
| 553 | } | ||
| 554 | |||
| 555 | private concatFiles (input: string, output: string) { | ||
| 556 | return new Promise<void>((res, rej) => { | ||
| 557 | const inputStream = createReadStream(input) | ||
| 558 | const outputStream = createWriteStream(output, { flags: 'a' }) | ||
| 559 | |||
| 560 | inputStream.pipe(outputStream) | ||
| 561 | |||
| 562 | inputStream.on('end', () => res()) | ||
| 563 | inputStream.on('error', err => rej(err)) | ||
| 564 | }) | ||
| 565 | } | ||
| 566 | |||
| 567 | private async regeneratePluginGlobalCSS () { | ||
| 568 | await this.resetCSSGlobalFile() | ||
| 569 | |||
| 570 | for (const plugin of this.getRegisteredPlugins()) { | ||
| 571 | await this.addCSSToGlobalFile(plugin.path, plugin.css) | ||
| 572 | } | ||
| 573 | } | ||
| 574 | |||
| 575 | // ###################### Utils ###################### | ||
| 576 | |||
| 577 | private sortHooksByPriority () { | ||
| 578 | for (const hookName of Object.keys(this.hooks)) { | ||
| 579 | this.hooks[hookName].sort((a, b) => { | ||
| 580 | return b.priority - a.priority | ||
| 581 | }) | ||
| 582 | } | ||
| 583 | } | ||
| 584 | |||
| 585 | private getPackageJSON (pluginName: string, pluginType: PluginType) { | ||
| 586 | const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json') | ||
| 587 | |||
| 588 | return readJSON(pluginPath) as Promise<PluginPackageJSON> | ||
| 589 | } | ||
| 590 | |||
| 591 | private getPluginPath (pluginName: string, pluginType: PluginType) { | ||
| 592 | const npmName = PluginModel.buildNpmName(pluginName, pluginType) | ||
| 593 | |||
| 594 | return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', npmName) | ||
| 595 | } | ||
| 596 | |||
| 597 | private getAuth (npmName: string, authName: string) { | ||
| 598 | const plugin = this.getRegisteredPluginOrTheme(npmName) | ||
| 599 | if (!plugin || plugin.type !== PluginType.PLUGIN) return null | ||
| 600 | |||
| 601 | let auths: (RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions)[] = plugin.registerHelpers.getIdAndPassAuths() | ||
| 602 | auths = auths.concat(plugin.registerHelpers.getExternalAuths()) | ||
| 603 | |||
| 604 | return auths.find(a => a.authName === authName) | ||
| 605 | } | ||
| 606 | |||
| 607 | // ###################### Private getters ###################### | ||
| 608 | |||
| 609 | private getRegisteredPluginsOrThemes (type: PluginType) { | ||
| 610 | const plugins: RegisteredPlugin[] = [] | ||
| 611 | |||
| 612 | for (const npmName of Object.keys(this.registeredPlugins)) { | ||
| 613 | const plugin = this.registeredPlugins[npmName] | ||
| 614 | if (plugin.type !== type) continue | ||
| 615 | |||
| 616 | plugins.push(plugin) | ||
| 617 | } | ||
| 618 | |||
| 619 | return plugins | ||
| 620 | } | ||
| 621 | |||
| 622 | // ###################### Generate register helpers ###################### | ||
| 623 | |||
| 624 | private getRegisterHelpers ( | ||
| 625 | npmName: string, | ||
| 626 | plugin: PluginModel | ||
| 627 | ): { registerStore: RegisterHelpers, registerOptions: RegisterServerOptions } { | ||
| 628 | const onHookAdded = (options: RegisterServerHookOptions) => { | ||
| 629 | if (!this.hooks[options.target]) this.hooks[options.target] = [] | ||
| 630 | |||
| 631 | this.hooks[options.target].push({ | ||
| 632 | npmName, | ||
| 633 | pluginName: plugin.name, | ||
| 634 | handler: options.handler, | ||
| 635 | priority: options.priority || 0 | ||
| 636 | }) | ||
| 637 | } | ||
| 638 | |||
| 639 | const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this)) | ||
| 640 | |||
| 641 | return { | ||
| 642 | registerStore: registerHelpers, | ||
| 643 | registerOptions: registerHelpers.buildRegisterHelpers() | ||
| 644 | } | ||
| 645 | } | ||
| 646 | |||
| 647 | private sanitizeAndCheckPackageJSONOrThrow (packageJSON: PluginPackageJSON, pluginType: PluginType) { | ||
| 648 | if (!packageJSON.staticDirs) packageJSON.staticDirs = {} | ||
| 649 | if (!packageJSON.css) packageJSON.css = [] | ||
| 650 | if (!packageJSON.clientScripts) packageJSON.clientScripts = [] | ||
| 651 | if (!packageJSON.translations) packageJSON.translations = {} | ||
| 652 | |||
| 653 | const { result: packageJSONValid, badFields } = isPackageJSONValid(packageJSON, pluginType) | ||
| 654 | if (!packageJSONValid) { | ||
| 655 | const formattedFields = badFields.map(f => `"${f}"`) | ||
| 656 | .join(', ') | ||
| 657 | |||
| 658 | throw new Error(`PackageJSON is invalid (invalid fields: ${formattedFields}).`) | ||
| 659 | } | ||
| 660 | } | ||
| 661 | |||
| 662 | static get Instance () { | ||
| 663 | return this.instance || (this.instance = new this()) | ||
| 664 | } | ||
| 665 | } | ||
diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts deleted file mode 100644 index 1aaef3606..000000000 --- a/server/lib/plugins/register-helpers.ts +++ /dev/null | |||
| @@ -1,340 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { Server } from 'http' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' | ||
| 5 | import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' | ||
| 6 | import { PluginModel } from '@server/models/server/plugin' | ||
| 7 | import { | ||
| 8 | RegisterServerAuthExternalOptions, | ||
| 9 | RegisterServerAuthExternalResult, | ||
| 10 | RegisterServerAuthPassOptions, | ||
| 11 | RegisterServerExternalAuthenticatedResult, | ||
| 12 | RegisterServerOptions, | ||
| 13 | RegisterServerWebSocketRouteOptions | ||
| 14 | } from '@server/types/plugins' | ||
| 15 | import { | ||
| 16 | EncoderOptionsBuilder, | ||
| 17 | PluginSettingsManager, | ||
| 18 | PluginStorageManager, | ||
| 19 | RegisterServerHookOptions, | ||
| 20 | RegisterServerSettingOptions, | ||
| 21 | serverHookObject, | ||
| 22 | SettingsChangeCallback, | ||
| 23 | VideoPlaylistPrivacy, | ||
| 24 | VideoPrivacy | ||
| 25 | } from '@shared/models' | ||
| 26 | import { VideoTranscodingProfilesManager } from '../transcoding/default-transcoding-profiles' | ||
| 27 | import { buildPluginHelpers } from './plugin-helpers-builder' | ||
| 28 | |||
| 29 | export class RegisterHelpers { | ||
| 30 | private readonly transcodingProfiles: { | ||
| 31 | [ npmName: string ]: { | ||
| 32 | type: 'vod' | 'live' | ||
| 33 | encoder: string | ||
| 34 | profile: string | ||
| 35 | }[] | ||
| 36 | } = {} | ||
| 37 | |||
| 38 | private readonly transcodingEncoders: { | ||
| 39 | [ npmName: string ]: { | ||
| 40 | type: 'vod' | 'live' | ||
| 41 | streamType: 'audio' | 'video' | ||
| 42 | encoder: string | ||
| 43 | priority: number | ||
| 44 | }[] | ||
| 45 | } = {} | ||
| 46 | |||
| 47 | private readonly settings: RegisterServerSettingOptions[] = [] | ||
| 48 | |||
| 49 | private idAndPassAuths: RegisterServerAuthPassOptions[] = [] | ||
| 50 | private externalAuths: RegisterServerAuthExternalOptions[] = [] | ||
| 51 | |||
| 52 | private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] | ||
| 53 | |||
| 54 | private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = [] | ||
| 55 | |||
| 56 | private readonly router: express.Router | ||
| 57 | private readonly videoConstantManagerFactory: VideoConstantManagerFactory | ||
| 58 | |||
| 59 | constructor ( | ||
| 60 | private readonly npmName: string, | ||
| 61 | private readonly plugin: PluginModel, | ||
| 62 | private readonly server: Server, | ||
| 63 | private readonly onHookAdded: (options: RegisterServerHookOptions) => void | ||
| 64 | ) { | ||
| 65 | this.router = express.Router() | ||
| 66 | this.videoConstantManagerFactory = new VideoConstantManagerFactory(this.npmName) | ||
| 67 | } | ||
| 68 | |||
| 69 | buildRegisterHelpers (): RegisterServerOptions { | ||
| 70 | const registerHook = this.buildRegisterHook() | ||
| 71 | const registerSetting = this.buildRegisterSetting() | ||
| 72 | |||
| 73 | const getRouter = this.buildGetRouter() | ||
| 74 | const registerWebSocketRoute = this.buildRegisterWebSocketRoute() | ||
| 75 | |||
| 76 | const settingsManager = this.buildSettingsManager() | ||
| 77 | const storageManager = this.buildStorageManager() | ||
| 78 | |||
| 79 | const videoLanguageManager = this.videoConstantManagerFactory.createVideoConstantManager<string>('language') | ||
| 80 | |||
| 81 | const videoLicenceManager = this.videoConstantManagerFactory.createVideoConstantManager<number>('licence') | ||
| 82 | const videoCategoryManager = this.videoConstantManagerFactory.createVideoConstantManager<number>('category') | ||
| 83 | |||
| 84 | const videoPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager<VideoPrivacy>('privacy') | ||
| 85 | const playlistPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager<VideoPlaylistPrivacy>('playlistPrivacy') | ||
| 86 | |||
| 87 | const transcodingManager = this.buildTranscodingManager() | ||
| 88 | |||
| 89 | const registerIdAndPassAuth = this.buildRegisterIdAndPassAuth() | ||
| 90 | const registerExternalAuth = this.buildRegisterExternalAuth() | ||
| 91 | const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() | ||
| 92 | const unregisterExternalAuth = this.buildUnregisterExternalAuth() | ||
| 93 | |||
| 94 | const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName) | ||
| 95 | |||
| 96 | return { | ||
| 97 | registerHook, | ||
| 98 | registerSetting, | ||
| 99 | |||
| 100 | getRouter, | ||
| 101 | registerWebSocketRoute, | ||
| 102 | |||
| 103 | settingsManager, | ||
| 104 | storageManager, | ||
| 105 | |||
| 106 | videoLanguageManager: { | ||
| 107 | ...videoLanguageManager, | ||
| 108 | /** @deprecated use `addConstant` instead **/ | ||
| 109 | addLanguage: videoLanguageManager.addConstant, | ||
| 110 | /** @deprecated use `deleteConstant` instead **/ | ||
| 111 | deleteLanguage: videoLanguageManager.deleteConstant | ||
| 112 | }, | ||
| 113 | videoCategoryManager: { | ||
| 114 | ...videoCategoryManager, | ||
| 115 | /** @deprecated use `addConstant` instead **/ | ||
| 116 | addCategory: videoCategoryManager.addConstant, | ||
| 117 | /** @deprecated use `deleteConstant` instead **/ | ||
| 118 | deleteCategory: videoCategoryManager.deleteConstant | ||
| 119 | }, | ||
| 120 | videoLicenceManager: { | ||
| 121 | ...videoLicenceManager, | ||
| 122 | /** @deprecated use `addConstant` instead **/ | ||
| 123 | addLicence: videoLicenceManager.addConstant, | ||
| 124 | /** @deprecated use `deleteConstant` instead **/ | ||
| 125 | deleteLicence: videoLicenceManager.deleteConstant | ||
| 126 | }, | ||
| 127 | |||
| 128 | videoPrivacyManager: { | ||
| 129 | ...videoPrivacyManager, | ||
| 130 | /** @deprecated use `deleteConstant` instead **/ | ||
| 131 | deletePrivacy: videoPrivacyManager.deleteConstant | ||
| 132 | }, | ||
| 133 | playlistPrivacyManager: { | ||
| 134 | ...playlistPrivacyManager, | ||
| 135 | /** @deprecated use `deleteConstant` instead **/ | ||
| 136 | deletePlaylistPrivacy: playlistPrivacyManager.deleteConstant | ||
| 137 | }, | ||
| 138 | |||
| 139 | transcodingManager, | ||
| 140 | |||
| 141 | registerIdAndPassAuth, | ||
| 142 | registerExternalAuth, | ||
| 143 | unregisterIdAndPassAuth, | ||
| 144 | unregisterExternalAuth, | ||
| 145 | |||
| 146 | peertubeHelpers | ||
| 147 | } | ||
| 148 | } | ||
| 149 | |||
| 150 | reinitVideoConstants (npmName: string) { | ||
| 151 | this.videoConstantManagerFactory.resetVideoConstants(npmName) | ||
| 152 | } | ||
| 153 | |||
| 154 | reinitTranscodingProfilesAndEncoders (npmName: string) { | ||
| 155 | const profiles = this.transcodingProfiles[npmName] | ||
| 156 | if (Array.isArray(profiles)) { | ||
| 157 | for (const profile of profiles) { | ||
| 158 | VideoTranscodingProfilesManager.Instance.removeProfile(profile) | ||
| 159 | } | ||
| 160 | } | ||
| 161 | |||
| 162 | const encoders = this.transcodingEncoders[npmName] | ||
| 163 | if (Array.isArray(encoders)) { | ||
| 164 | for (const o of encoders) { | ||
| 165 | VideoTranscodingProfilesManager.Instance.removeEncoderPriority(o.type, o.streamType, o.encoder, o.priority) | ||
| 166 | } | ||
| 167 | } | ||
| 168 | } | ||
| 169 | |||
| 170 | getSettings () { | ||
| 171 | return this.settings | ||
| 172 | } | ||
| 173 | |||
| 174 | getRouter () { | ||
| 175 | return this.router | ||
| 176 | } | ||
| 177 | |||
| 178 | getIdAndPassAuths () { | ||
| 179 | return this.idAndPassAuths | ||
| 180 | } | ||
| 181 | |||
| 182 | getExternalAuths () { | ||
| 183 | return this.externalAuths | ||
| 184 | } | ||
| 185 | |||
| 186 | getOnSettingsChangedCallbacks () { | ||
| 187 | return this.onSettingsChangeCallbacks | ||
| 188 | } | ||
| 189 | |||
| 190 | getWebSocketRoutes () { | ||
| 191 | return this.webSocketRoutes | ||
| 192 | } | ||
| 193 | |||
| 194 | private buildGetRouter () { | ||
| 195 | return () => this.router | ||
| 196 | } | ||
| 197 | |||
| 198 | private buildRegisterWebSocketRoute () { | ||
| 199 | return (options: RegisterServerWebSocketRouteOptions) => { | ||
| 200 | this.webSocketRoutes.push(options) | ||
| 201 | } | ||
| 202 | } | ||
| 203 | |||
| 204 | private buildRegisterSetting () { | ||
| 205 | return (options: RegisterServerSettingOptions) => { | ||
| 206 | this.settings.push(options) | ||
| 207 | } | ||
| 208 | } | ||
| 209 | |||
| 210 | private buildRegisterHook () { | ||
| 211 | return (options: RegisterServerHookOptions) => { | ||
| 212 | if (serverHookObject[options.target] !== true) { | ||
| 213 | logger.warn('Unknown hook %s of plugin %s. Skipping.', options.target, this.npmName) | ||
| 214 | return | ||
| 215 | } | ||
| 216 | |||
| 217 | return this.onHookAdded(options) | ||
| 218 | } | ||
| 219 | } | ||
| 220 | |||
| 221 | private buildRegisterIdAndPassAuth () { | ||
| 222 | return (options: RegisterServerAuthPassOptions) => { | ||
| 223 | if (!options.authName || typeof options.getWeight !== 'function' || typeof options.login !== 'function') { | ||
| 224 | logger.error('Cannot register auth plugin %s: authName, getWeight or login are not valid.', this.npmName, { options }) | ||
| 225 | return | ||
| 226 | } | ||
| 227 | |||
| 228 | this.idAndPassAuths.push(options) | ||
| 229 | } | ||
| 230 | } | ||
| 231 | |||
| 232 | private buildRegisterExternalAuth () { | ||
| 233 | const self = this | ||
| 234 | |||
| 235 | return (options: RegisterServerAuthExternalOptions) => { | ||
| 236 | if (!options.authName || typeof options.authDisplayName !== 'function' || typeof options.onAuthRequest !== 'function') { | ||
| 237 | logger.error('Cannot register auth plugin %s: authName, authDisplayName or onAuthRequest are not valid.', this.npmName, { options }) | ||
| 238 | return | ||
| 239 | } | ||
| 240 | |||
| 241 | this.externalAuths.push(options) | ||
| 242 | |||
| 243 | return { | ||
| 244 | userAuthenticated (result: RegisterServerExternalAuthenticatedResult): void { | ||
| 245 | onExternalUserAuthenticated({ | ||
| 246 | npmName: self.npmName, | ||
| 247 | authName: options.authName, | ||
| 248 | authResult: result | ||
| 249 | }).catch(err => { | ||
| 250 | logger.error('Cannot execute onExternalUserAuthenticated.', { npmName: self.npmName, authName: options.authName, err }) | ||
| 251 | }) | ||
| 252 | } | ||
| 253 | } as RegisterServerAuthExternalResult | ||
| 254 | } | ||
| 255 | } | ||
| 256 | |||
| 257 | private buildUnregisterExternalAuth () { | ||
| 258 | return (authName: string) => { | ||
| 259 | this.externalAuths = this.externalAuths.filter(a => a.authName !== authName) | ||
| 260 | } | ||
| 261 | } | ||
| 262 | |||
| 263 | private buildUnregisterIdAndPassAuth () { | ||
| 264 | return (authName: string) => { | ||
| 265 | this.idAndPassAuths = this.idAndPassAuths.filter(a => a.authName !== authName) | ||
| 266 | } | ||
| 267 | } | ||
| 268 | |||
| 269 | private buildSettingsManager (): PluginSettingsManager { | ||
| 270 | return { | ||
| 271 | getSetting: (name: string) => PluginModel.getSetting(this.plugin.name, this.plugin.type, name, this.settings), | ||
| 272 | |||
| 273 | getSettings: (names: string[]) => PluginModel.getSettings(this.plugin.name, this.plugin.type, names, this.settings), | ||
| 274 | |||
| 275 | setSetting: (name: string, value: string) => PluginModel.setSetting(this.plugin.name, this.plugin.type, name, value), | ||
| 276 | |||
| 277 | onSettingsChange: (cb: SettingsChangeCallback) => this.onSettingsChangeCallbacks.push(cb) | ||
| 278 | } | ||
| 279 | } | ||
| 280 | |||
| 281 | private buildStorageManager (): PluginStorageManager { | ||
| 282 | return { | ||
| 283 | getData: (key: string) => PluginModel.getData(this.plugin.name, this.plugin.type, key), | ||
| 284 | |||
| 285 | storeData: (key: string, data: any) => PluginModel.storeData(this.plugin.name, this.plugin.type, key, data) | ||
| 286 | } | ||
| 287 | } | ||
| 288 | |||
| 289 | private buildTranscodingManager () { | ||
| 290 | const self = this | ||
| 291 | |||
| 292 | function addProfile (type: 'live' | 'vod', encoder: string, profile: string, builder: EncoderOptionsBuilder) { | ||
| 293 | if (profile === 'default') { | ||
| 294 | logger.error('A plugin cannot add a default live transcoding profile') | ||
| 295 | return false | ||
| 296 | } | ||
| 297 | |||
| 298 | VideoTranscodingProfilesManager.Instance.addProfile({ | ||
| 299 | type, | ||
| 300 | encoder, | ||
| 301 | profile, | ||
| 302 | builder | ||
| 303 | }) | ||
| 304 | |||
| 305 | if (!self.transcodingProfiles[self.npmName]) self.transcodingProfiles[self.npmName] = [] | ||
| 306 | self.transcodingProfiles[self.npmName].push({ type, encoder, profile }) | ||
| 307 | |||
| 308 | return true | ||
| 309 | } | ||
| 310 | |||
| 311 | function addEncoderPriority (type: 'live' | 'vod', streamType: 'audio' | 'video', encoder: string, priority: number) { | ||
| 312 | VideoTranscodingProfilesManager.Instance.addEncoderPriority(type, streamType, encoder, priority) | ||
| 313 | |||
| 314 | if (!self.transcodingEncoders[self.npmName]) self.transcodingEncoders[self.npmName] = [] | ||
| 315 | self.transcodingEncoders[self.npmName].push({ type, streamType, encoder, priority }) | ||
| 316 | } | ||
| 317 | |||
| 318 | return { | ||
| 319 | addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { | ||
| 320 | return addProfile('live', encoder, profile, builder) | ||
| 321 | }, | ||
| 322 | |||
| 323 | addVODProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { | ||
| 324 | return addProfile('vod', encoder, profile, builder) | ||
| 325 | }, | ||
| 326 | |||
| 327 | addLiveEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { | ||
| 328 | return addEncoderPriority('live', streamType, encoder, priority) | ||
| 329 | }, | ||
| 330 | |||
| 331 | addVODEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { | ||
| 332 | return addEncoderPriority('vod', streamType, encoder, priority) | ||
| 333 | }, | ||
| 334 | |||
| 335 | removeAllProfilesAndEncoderPriorities () { | ||
| 336 | return self.reinitTranscodingProfilesAndEncoders(self.npmName) | ||
| 337 | } | ||
| 338 | } | ||
| 339 | } | ||
| 340 | } | ||
diff --git a/server/lib/plugins/theme-utils.ts b/server/lib/plugins/theme-utils.ts deleted file mode 100644 index 76c671f1c..000000000 --- a/server/lib/plugins/theme-utils.ts +++ /dev/null | |||
| @@ -1,24 +0,0 @@ | |||
| 1 | import { DEFAULT_THEME_NAME, DEFAULT_USER_THEME_NAME } from '../../initializers/constants' | ||
| 2 | import { PluginManager } from './plugin-manager' | ||
| 3 | import { CONFIG } from '../../initializers/config' | ||
| 4 | |||
| 5 | function getThemeOrDefault (name: string, defaultTheme: string) { | ||
| 6 | if (isThemeRegistered(name)) return name | ||
| 7 | |||
| 8 | // Fallback to admin default theme | ||
| 9 | if (name !== CONFIG.THEME.DEFAULT) return getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) | ||
| 10 | |||
| 11 | return defaultTheme | ||
| 12 | } | ||
| 13 | |||
| 14 | function isThemeRegistered (name: string) { | ||
| 15 | if (name === DEFAULT_THEME_NAME || name === DEFAULT_USER_THEME_NAME) return true | ||
| 16 | |||
| 17 | return !!PluginManager.Instance.getRegisteredThemes() | ||
| 18 | .find(r => r.name === name) | ||
| 19 | } | ||
| 20 | |||
| 21 | export { | ||
| 22 | getThemeOrDefault, | ||
| 23 | isThemeRegistered | ||
| 24 | } | ||
diff --git a/server/lib/plugins/video-constant-manager-factory.ts b/server/lib/plugins/video-constant-manager-factory.ts deleted file mode 100644 index 5f7edfbe2..000000000 --- a/server/lib/plugins/video-constant-manager-factory.ts +++ /dev/null | |||
| @@ -1,139 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { | ||
| 3 | VIDEO_CATEGORIES, | ||
| 4 | VIDEO_LANGUAGES, | ||
| 5 | VIDEO_LICENCES, | ||
| 6 | VIDEO_PLAYLIST_PRIVACIES, | ||
| 7 | VIDEO_PRIVACIES | ||
| 8 | } from '@server/initializers/constants' | ||
| 9 | import { ConstantManager } from '@shared/models/plugins/server/plugin-constant-manager.model' | ||
| 10 | |||
| 11 | type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy' | ||
| 12 | type VideoConstant = Record<number | string, string> | ||
| 13 | |||
| 14 | type UpdatedVideoConstant = { | ||
| 15 | [name in AlterableVideoConstant]: { | ||
| 16 | [ npmName: string]: { | ||
| 17 | added: VideoConstant[] | ||
| 18 | deleted: VideoConstant[] | ||
| 19 | } | ||
| 20 | } | ||
| 21 | } | ||
| 22 | |||
| 23 | const constantsHash: { [key in AlterableVideoConstant]: VideoConstant } = { | ||
| 24 | language: VIDEO_LANGUAGES, | ||
| 25 | licence: VIDEO_LICENCES, | ||
| 26 | category: VIDEO_CATEGORIES, | ||
| 27 | privacy: VIDEO_PRIVACIES, | ||
| 28 | playlistPrivacy: VIDEO_PLAYLIST_PRIVACIES | ||
| 29 | } | ||
| 30 | |||
| 31 | export class VideoConstantManagerFactory { | ||
| 32 | private readonly updatedVideoConstants: UpdatedVideoConstant = { | ||
| 33 | playlistPrivacy: { }, | ||
| 34 | privacy: { }, | ||
| 35 | language: { }, | ||
| 36 | licence: { }, | ||
| 37 | category: { } | ||
| 38 | } | ||
| 39 | |||
| 40 | constructor ( | ||
| 41 | private readonly npmName: string | ||
| 42 | ) {} | ||
| 43 | |||
| 44 | public resetVideoConstants (npmName: string) { | ||
| 45 | const types: AlterableVideoConstant[] = [ 'language', 'licence', 'category', 'privacy', 'playlistPrivacy' ] | ||
| 46 | for (const type of types) { | ||
| 47 | this.resetConstants({ npmName, type }) | ||
| 48 | } | ||
| 49 | } | ||
| 50 | |||
| 51 | private resetConstants (parameters: { npmName: string, type: AlterableVideoConstant }) { | ||
| 52 | const { npmName, type } = parameters | ||
| 53 | const updatedConstants = this.updatedVideoConstants[type][npmName] | ||
| 54 | |||
| 55 | if (!updatedConstants) return | ||
| 56 | |||
| 57 | for (const added of updatedConstants.added) { | ||
| 58 | delete constantsHash[type][added.key] | ||
| 59 | } | ||
| 60 | |||
| 61 | for (const deleted of updatedConstants.deleted) { | ||
| 62 | constantsHash[type][deleted.key] = deleted.label | ||
| 63 | } | ||
| 64 | |||
| 65 | delete this.updatedVideoConstants[type][npmName] | ||
| 66 | } | ||
| 67 | |||
| 68 | public createVideoConstantManager<K extends number | string>(type: AlterableVideoConstant): ConstantManager<K> { | ||
| 69 | const { npmName } = this | ||
| 70 | return { | ||
| 71 | addConstant: (key: K, label: string) => this.addConstant({ npmName, type, key, label }), | ||
| 72 | deleteConstant: (key: K) => this.deleteConstant({ npmName, type, key }), | ||
| 73 | getConstantValue: (key: K) => constantsHash[type][key], | ||
| 74 | getConstants: () => constantsHash[type] as Record<K, string>, | ||
| 75 | resetConstants: () => this.resetConstants({ npmName, type }) | ||
| 76 | } | ||
| 77 | } | ||
| 78 | |||
| 79 | private addConstant<T extends string | number> (parameters: { | ||
| 80 | npmName: string | ||
| 81 | type: AlterableVideoConstant | ||
| 82 | key: T | ||
| 83 | label: string | ||
| 84 | }) { | ||
| 85 | const { npmName, type, key, label } = parameters | ||
| 86 | const obj = constantsHash[type] | ||
| 87 | |||
| 88 | if (obj[key]) { | ||
| 89 | logger.warn('Cannot add %s %s by plugin %s: key already exists.', type, npmName, key) | ||
| 90 | return false | ||
| 91 | } | ||
| 92 | |||
| 93 | if (!this.updatedVideoConstants[type][npmName]) { | ||
| 94 | this.updatedVideoConstants[type][npmName] = { | ||
| 95 | added: [], | ||
| 96 | deleted: [] | ||
| 97 | } | ||
| 98 | } | ||
| 99 | |||
| 100 | this.updatedVideoConstants[type][npmName].added.push({ key, label } as VideoConstant) | ||
| 101 | obj[key] = label | ||
| 102 | |||
| 103 | return true | ||
| 104 | } | ||
| 105 | |||
| 106 | private deleteConstant<T extends string | number> (parameters: { | ||
| 107 | npmName: string | ||
| 108 | type: AlterableVideoConstant | ||
| 109 | key: T | ||
| 110 | }) { | ||
| 111 | const { npmName, type, key } = parameters | ||
| 112 | const obj = constantsHash[type] | ||
| 113 | |||
| 114 | if (!obj[key]) { | ||
| 115 | logger.warn('Cannot delete %s by plugin %s: key %s does not exist.', type, npmName, key) | ||
| 116 | return false | ||
| 117 | } | ||
| 118 | |||
| 119 | if (!this.updatedVideoConstants[type][npmName]) { | ||
| 120 | this.updatedVideoConstants[type][npmName] = { | ||
| 121 | added: [], | ||
| 122 | deleted: [] | ||
| 123 | } | ||
| 124 | } | ||
| 125 | |||
| 126 | const updatedConstants = this.updatedVideoConstants[type][npmName] | ||
| 127 | |||
| 128 | const alreadyAdded = updatedConstants.added.find(a => a.key === key) | ||
| 129 | if (alreadyAdded) { | ||
| 130 | updatedConstants.added.filter(a => a.key !== key) | ||
| 131 | } else if (obj[key]) { | ||
| 132 | updatedConstants.deleted.push({ key, label: obj[key] } as VideoConstant) | ||
| 133 | } | ||
| 134 | |||
| 135 | delete obj[key] | ||
| 136 | |||
| 137 | return true | ||
| 138 | } | ||
| 139 | } | ||
diff --git a/server/lib/plugins/yarn.ts b/server/lib/plugins/yarn.ts deleted file mode 100644 index 9cf6ec9e9..000000000 --- a/server/lib/plugins/yarn.ts +++ /dev/null | |||
| @@ -1,73 +0,0 @@ | |||
| 1 | import { outputJSON, pathExists } from 'fs-extra' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { execShell } from '../../helpers/core-utils' | ||
| 4 | import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins' | ||
| 5 | import { logger } from '../../helpers/logger' | ||
| 6 | import { CONFIG } from '../../initializers/config' | ||
| 7 | import { getLatestPluginVersion } from './plugin-index' | ||
| 8 | |||
| 9 | async function installNpmPlugin (npmName: string, versionArg?: string) { | ||
| 10 | // Security check | ||
| 11 | checkNpmPluginNameOrThrow(npmName) | ||
| 12 | if (versionArg) checkPluginVersionOrThrow(versionArg) | ||
| 13 | |||
| 14 | const version = versionArg || await getLatestPluginVersion(npmName) | ||
| 15 | |||
| 16 | let toInstall = npmName | ||
| 17 | if (version) toInstall += `@${version}` | ||
| 18 | |||
| 19 | const { stdout } = await execYarn('add ' + toInstall) | ||
| 20 | |||
| 21 | logger.debug('Added a yarn package.', { yarnStdout: stdout }) | ||
| 22 | } | ||
| 23 | |||
| 24 | async function installNpmPluginFromDisk (path: string) { | ||
| 25 | await execYarn('add file:' + path) | ||
| 26 | } | ||
| 27 | |||
| 28 | async function removeNpmPlugin (name: string) { | ||
| 29 | checkNpmPluginNameOrThrow(name) | ||
| 30 | |||
| 31 | await execYarn('remove ' + name) | ||
| 32 | } | ||
| 33 | |||
| 34 | async function rebuildNativePlugins () { | ||
| 35 | await execYarn('install --pure-lockfile') | ||
| 36 | } | ||
| 37 | |||
| 38 | // ############################################################################ | ||
| 39 | |||
| 40 | export { | ||
| 41 | installNpmPlugin, | ||
| 42 | installNpmPluginFromDisk, | ||
| 43 | rebuildNativePlugins, | ||
| 44 | removeNpmPlugin | ||
| 45 | } | ||
| 46 | |||
| 47 | // ############################################################################ | ||
| 48 | |||
| 49 | async function execYarn (command: string) { | ||
| 50 | try { | ||
| 51 | const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR | ||
| 52 | const pluginPackageJSON = join(pluginDirectory, 'package.json') | ||
| 53 | |||
| 54 | // Create empty package.json file if needed | ||
| 55 | if (!await pathExists(pluginPackageJSON)) { | ||
| 56 | await outputJSON(pluginPackageJSON, {}) | ||
| 57 | } | ||
| 58 | |||
| 59 | return execShell(`yarn ${command}`, { cwd: pluginDirectory }) | ||
| 60 | } catch (result) { | ||
| 61 | logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr }) | ||
| 62 | |||
| 63 | throw result.err | ||
| 64 | } | ||
| 65 | } | ||
| 66 | |||
| 67 | function checkNpmPluginNameOrThrow (name: string) { | ||
| 68 | if (!isNpmPluginNameValid(name)) throw new Error('Invalid NPM plugin name to install') | ||
| 69 | } | ||
| 70 | |||
| 71 | function checkPluginVersionOrThrow (name: string) { | ||
| 72 | if (!isPluginStableOrUnstableVersionValid(name)) throw new Error('Invalid NPM plugin version to install') | ||
| 73 | } | ||
diff --git a/server/lib/redis.ts b/server/lib/redis.ts deleted file mode 100644 index 48d9986b5..000000000 --- a/server/lib/redis.ts +++ /dev/null | |||
| @@ -1,465 +0,0 @@ | |||
| 1 | import IoRedis, { RedisOptions } from 'ioredis' | ||
| 2 | import { exists } from '@server/helpers/custom-validators/misc' | ||
| 3 | import { sha256 } from '@shared/extra-utils' | ||
| 4 | import { logger } from '../helpers/logger' | ||
| 5 | import { generateRandomString } from '../helpers/utils' | ||
| 6 | import { CONFIG } from '../initializers/config' | ||
| 7 | import { | ||
| 8 | AP_CLEANER, | ||
| 9 | CONTACT_FORM_LIFETIME, | ||
| 10 | RESUMABLE_UPLOAD_SESSION_LIFETIME, | ||
| 11 | TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, | ||
| 12 | EMAIL_VERIFY_LIFETIME, | ||
| 13 | USER_PASSWORD_CREATE_LIFETIME, | ||
| 14 | USER_PASSWORD_RESET_LIFETIME, | ||
| 15 | VIEW_LIFETIME, | ||
| 16 | WEBSERVER | ||
| 17 | } from '../initializers/constants' | ||
| 18 | |||
| 19 | class Redis { | ||
| 20 | |||
| 21 | private static instance: Redis | ||
| 22 | private initialized = false | ||
| 23 | private connected = false | ||
| 24 | private client: IoRedis | ||
| 25 | private prefix: string | ||
| 26 | |||
| 27 | private constructor () { | ||
| 28 | } | ||
| 29 | |||
| 30 | init () { | ||
| 31 | // Already initialized | ||
| 32 | if (this.initialized === true) return | ||
| 33 | this.initialized = true | ||
| 34 | |||
| 35 | const redisMode = CONFIG.REDIS.SENTINEL.ENABLED ? 'sentinel' : 'standalone' | ||
| 36 | logger.info('Connecting to redis ' + redisMode + '...') | ||
| 37 | |||
| 38 | this.client = new IoRedis(Redis.getRedisClientOptions('', { enableAutoPipelining: true })) | ||
| 39 | this.client.on('error', err => logger.error('Redis failed to connect', { err })) | ||
| 40 | this.client.on('connect', () => { | ||
| 41 | logger.info('Connected to redis.') | ||
| 42 | |||
| 43 | this.connected = true | ||
| 44 | }) | ||
| 45 | this.client.on('reconnecting', (ms) => { | ||
| 46 | logger.error(`Reconnecting to redis in ${ms}.`) | ||
| 47 | }) | ||
| 48 | this.client.on('close', () => { | ||
| 49 | logger.error('Connection to redis has closed.') | ||
| 50 | this.connected = false | ||
| 51 | }) | ||
| 52 | |||
| 53 | this.client.on('end', () => { | ||
| 54 | logger.error('Connection to redis has closed and no more reconnects will be done.') | ||
| 55 | }) | ||
| 56 | |||
| 57 | this.prefix = 'redis-' + WEBSERVER.HOST + '-' | ||
| 58 | } | ||
| 59 | |||
| 60 | static getRedisClientOptions (name?: string, options: RedisOptions = {}): RedisOptions { | ||
| 61 | const connectionName = [ 'PeerTube', name ].join('') | ||
| 62 | const connectTimeout = 20000 // Could be slow since node use sync call to compile PeerTube | ||
| 63 | |||
| 64 | if (CONFIG.REDIS.SENTINEL.ENABLED) { | ||
| 65 | return { | ||
| 66 | connectionName, | ||
| 67 | connectTimeout, | ||
| 68 | enableTLSForSentinelMode: CONFIG.REDIS.SENTINEL.ENABLE_TLS, | ||
| 69 | sentinelPassword: CONFIG.REDIS.AUTH, | ||
| 70 | sentinels: CONFIG.REDIS.SENTINEL.SENTINELS, | ||
| 71 | name: CONFIG.REDIS.SENTINEL.MASTER_NAME, | ||
| 72 | ...options | ||
| 73 | } | ||
| 74 | } | ||
| 75 | |||
| 76 | return { | ||
| 77 | connectionName, | ||
| 78 | connectTimeout, | ||
| 79 | password: CONFIG.REDIS.AUTH, | ||
| 80 | db: CONFIG.REDIS.DB, | ||
| 81 | host: CONFIG.REDIS.HOSTNAME, | ||
| 82 | port: CONFIG.REDIS.PORT, | ||
| 83 | path: CONFIG.REDIS.SOCKET, | ||
| 84 | showFriendlyErrorStack: true, | ||
| 85 | ...options | ||
| 86 | } | ||
| 87 | } | ||
| 88 | |||
| 89 | getClient () { | ||
| 90 | return this.client | ||
| 91 | } | ||
| 92 | |||
| 93 | getPrefix () { | ||
| 94 | return this.prefix | ||
| 95 | } | ||
| 96 | |||
| 97 | isConnected () { | ||
| 98 | return this.connected | ||
| 99 | } | ||
| 100 | |||
| 101 | /* ************ Forgot password ************ */ | ||
| 102 | |||
| 103 | async setResetPasswordVerificationString (userId: number) { | ||
| 104 | const generatedString = await generateRandomString(32) | ||
| 105 | |||
| 106 | await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_RESET_LIFETIME) | ||
| 107 | |||
| 108 | return generatedString | ||
| 109 | } | ||
| 110 | |||
| 111 | async setCreatePasswordVerificationString (userId: number) { | ||
| 112 | const generatedString = await generateRandomString(32) | ||
| 113 | |||
| 114 | await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_CREATE_LIFETIME) | ||
| 115 | |||
| 116 | return generatedString | ||
| 117 | } | ||
| 118 | |||
| 119 | async removePasswordVerificationString (userId: number) { | ||
| 120 | return this.removeValue(this.generateResetPasswordKey(userId)) | ||
| 121 | } | ||
| 122 | |||
| 123 | async getResetPasswordVerificationString (userId: number) { | ||
| 124 | return this.getValue(this.generateResetPasswordKey(userId)) | ||
| 125 | } | ||
| 126 | |||
| 127 | /* ************ Two factor auth request ************ */ | ||
| 128 | |||
| 129 | async setTwoFactorRequest (userId: number, otpSecret: string) { | ||
| 130 | const requestToken = await generateRandomString(32) | ||
| 131 | |||
| 132 | await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME) | ||
| 133 | |||
| 134 | return requestToken | ||
| 135 | } | ||
| 136 | |||
| 137 | async getTwoFactorRequestToken (userId: number, requestToken: string) { | ||
| 138 | return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken)) | ||
| 139 | } | ||
| 140 | |||
| 141 | /* ************ Email verification ************ */ | ||
| 142 | |||
| 143 | async setUserVerifyEmailVerificationString (userId: number) { | ||
| 144 | const generatedString = await generateRandomString(32) | ||
| 145 | |||
| 146 | await this.setValue(this.generateUserVerifyEmailKey(userId), generatedString, EMAIL_VERIFY_LIFETIME) | ||
| 147 | |||
| 148 | return generatedString | ||
| 149 | } | ||
| 150 | |||
| 151 | async getUserVerifyEmailLink (userId: number) { | ||
| 152 | return this.getValue(this.generateUserVerifyEmailKey(userId)) | ||
| 153 | } | ||
| 154 | |||
| 155 | async setRegistrationVerifyEmailVerificationString (registrationId: number) { | ||
| 156 | const generatedString = await generateRandomString(32) | ||
| 157 | |||
| 158 | await this.setValue(this.generateRegistrationVerifyEmailKey(registrationId), generatedString, EMAIL_VERIFY_LIFETIME) | ||
| 159 | |||
| 160 | return generatedString | ||
| 161 | } | ||
| 162 | |||
| 163 | async getRegistrationVerifyEmailLink (registrationId: number) { | ||
| 164 | return this.getValue(this.generateRegistrationVerifyEmailKey(registrationId)) | ||
| 165 | } | ||
| 166 | |||
| 167 | /* ************ Contact form per IP ************ */ | ||
| 168 | |||
| 169 | async setContactFormIp (ip: string) { | ||
| 170 | return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) | ||
| 171 | } | ||
| 172 | |||
| 173 | async doesContactFormIpExist (ip: string) { | ||
| 174 | return this.exists(this.generateContactFormKey(ip)) | ||
| 175 | } | ||
| 176 | |||
| 177 | /* ************ Views per IP ************ */ | ||
| 178 | |||
| 179 | setIPVideoView (ip: string, videoUUID: string) { | ||
| 180 | return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) | ||
| 181 | } | ||
| 182 | |||
| 183 | async doesVideoIPViewExist (ip: string, videoUUID: string) { | ||
| 184 | return this.exists(this.generateIPViewKey(ip, videoUUID)) | ||
| 185 | } | ||
| 186 | |||
| 187 | /* ************ Video views stats ************ */ | ||
| 188 | |||
| 189 | addVideoViewStats (videoId: number) { | ||
| 190 | const { videoKey, setKey } = this.generateVideoViewStatsKeys({ videoId }) | ||
| 191 | |||
| 192 | return Promise.all([ | ||
| 193 | this.addToSet(setKey, videoId.toString()), | ||
| 194 | this.increment(videoKey) | ||
| 195 | ]) | ||
| 196 | } | ||
| 197 | |||
| 198 | async getVideoViewsStats (videoId: number, hour: number) { | ||
| 199 | const { videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) | ||
| 200 | |||
| 201 | const valueString = await this.getValue(videoKey) | ||
| 202 | const valueInt = parseInt(valueString, 10) | ||
| 203 | |||
| 204 | if (isNaN(valueInt)) { | ||
| 205 | logger.error('Cannot get videos views stats of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString) | ||
| 206 | return undefined | ||
| 207 | } | ||
| 208 | |||
| 209 | return valueInt | ||
| 210 | } | ||
| 211 | |||
| 212 | async listVideosViewedForStats (hour: number) { | ||
| 213 | const { setKey } = this.generateVideoViewStatsKeys({ hour }) | ||
| 214 | |||
| 215 | const stringIds = await this.getSet(setKey) | ||
| 216 | return stringIds.map(s => parseInt(s, 10)) | ||
| 217 | } | ||
| 218 | |||
| 219 | deleteVideoViewsStats (videoId: number, hour: number) { | ||
| 220 | const { setKey, videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) | ||
| 221 | |||
| 222 | return Promise.all([ | ||
| 223 | this.deleteFromSet(setKey, videoId.toString()), | ||
| 224 | this.deleteKey(videoKey) | ||
| 225 | ]) | ||
| 226 | } | ||
| 227 | |||
| 228 | /* ************ Local video views buffer ************ */ | ||
| 229 | |||
| 230 | addLocalVideoView (videoId: number) { | ||
| 231 | const { videoKey, setKey } = this.generateLocalVideoViewsKeys(videoId) | ||
| 232 | |||
| 233 | return Promise.all([ | ||
| 234 | this.addToSet(setKey, videoId.toString()), | ||
| 235 | this.increment(videoKey) | ||
| 236 | ]) | ||
| 237 | } | ||
| 238 | |||
| 239 | async getLocalVideoViews (videoId: number) { | ||
| 240 | const { videoKey } = this.generateLocalVideoViewsKeys(videoId) | ||
| 241 | |||
| 242 | const valueString = await this.getValue(videoKey) | ||
| 243 | const valueInt = parseInt(valueString, 10) | ||
| 244 | |||
| 245 | if (isNaN(valueInt)) { | ||
| 246 | logger.error('Cannot get videos views of video %d: views number is NaN (%s).', videoId, valueString) | ||
| 247 | return undefined | ||
| 248 | } | ||
| 249 | |||
| 250 | return valueInt | ||
| 251 | } | ||
| 252 | |||
| 253 | async listLocalVideosViewed () { | ||
| 254 | const { setKey } = this.generateLocalVideoViewsKeys() | ||
| 255 | |||
| 256 | const stringIds = await this.getSet(setKey) | ||
| 257 | return stringIds.map(s => parseInt(s, 10)) | ||
| 258 | } | ||
| 259 | |||
| 260 | deleteLocalVideoViews (videoId: number) { | ||
| 261 | const { setKey, videoKey } = this.generateLocalVideoViewsKeys(videoId) | ||
| 262 | |||
| 263 | return Promise.all([ | ||
| 264 | this.deleteFromSet(setKey, videoId.toString()), | ||
| 265 | this.deleteKey(videoKey) | ||
| 266 | ]) | ||
| 267 | } | ||
| 268 | |||
| 269 | /* ************ Video viewers stats ************ */ | ||
| 270 | |||
| 271 | getLocalVideoViewer (options: { | ||
| 272 | key?: string | ||
| 273 | // Or | ||
| 274 | ip?: string | ||
| 275 | videoId?: number | ||
| 276 | }) { | ||
| 277 | if (options.key) return this.getObject(options.key) | ||
| 278 | |||
| 279 | const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId) | ||
| 280 | |||
| 281 | return this.getObject(viewerKey) | ||
| 282 | } | ||
| 283 | |||
| 284 | setLocalVideoViewer (ip: string, videoId: number, object: any) { | ||
| 285 | const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId) | ||
| 286 | |||
| 287 | return Promise.all([ | ||
| 288 | this.addToSet(setKey, viewerKey), | ||
| 289 | this.setObject(viewerKey, object) | ||
| 290 | ]) | ||
| 291 | } | ||
| 292 | |||
| 293 | listLocalVideoViewerKeys () { | ||
| 294 | const { setKey } = this.generateLocalVideoViewerKeys() | ||
| 295 | |||
| 296 | return this.getSet(setKey) | ||
| 297 | } | ||
| 298 | |||
| 299 | deleteLocalVideoViewersKeys (key: string) { | ||
| 300 | const { setKey } = this.generateLocalVideoViewerKeys() | ||
| 301 | |||
| 302 | return Promise.all([ | ||
| 303 | this.deleteFromSet(setKey, key), | ||
| 304 | this.deleteKey(key) | ||
| 305 | ]) | ||
| 306 | } | ||
| 307 | |||
| 308 | /* ************ Resumable uploads final responses ************ */ | ||
| 309 | |||
| 310 | setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) { | ||
| 311 | return this.setValue( | ||
| 312 | 'resumable-upload-' + uploadId, | ||
| 313 | response | ||
| 314 | ? JSON.stringify(response) | ||
| 315 | : '', | ||
| 316 | RESUMABLE_UPLOAD_SESSION_LIFETIME | ||
| 317 | ) | ||
| 318 | } | ||
| 319 | |||
| 320 | doesUploadSessionExist (uploadId: string) { | ||
| 321 | return this.exists('resumable-upload-' + uploadId) | ||
| 322 | } | ||
| 323 | |||
| 324 | async getUploadSession (uploadId: string) { | ||
| 325 | const value = await this.getValue('resumable-upload-' + uploadId) | ||
| 326 | |||
| 327 | return value | ||
| 328 | ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } } | ||
| 329 | : undefined | ||
| 330 | } | ||
| 331 | |||
| 332 | deleteUploadSession (uploadId: string) { | ||
| 333 | return this.deleteKey('resumable-upload-' + uploadId) | ||
| 334 | } | ||
| 335 | |||
| 336 | /* ************ AP resource unavailability ************ */ | ||
| 337 | |||
| 338 | async addAPUnavailability (url: string) { | ||
| 339 | const key = this.generateAPUnavailabilityKey(url) | ||
| 340 | |||
| 341 | const value = await this.increment(key) | ||
| 342 | await this.setExpiration(key, AP_CLEANER.PERIOD * 2) | ||
| 343 | |||
| 344 | return value | ||
| 345 | } | ||
| 346 | |||
| 347 | /* ************ Keys generation ************ */ | ||
| 348 | |||
| 349 | private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string } | ||
| 350 | private generateLocalVideoViewsKeys (): { setKey: string } | ||
| 351 | private generateLocalVideoViewsKeys (videoId?: number) { | ||
| 352 | return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } | ||
| 353 | } | ||
| 354 | |||
| 355 | private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string } | ||
| 356 | private generateLocalVideoViewerKeys (): { setKey: string } | ||
| 357 | private generateLocalVideoViewerKeys (ip?: string, videoId?: number) { | ||
| 358 | return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` } | ||
| 359 | } | ||
| 360 | |||
| 361 | private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { | ||
| 362 | const hour = exists(options.hour) | ||
| 363 | ? options.hour | ||
| 364 | : new Date().getHours() | ||
| 365 | |||
| 366 | return { setKey: `videos-view-h${hour}`, videoKey: `video-view-${options.videoId}-h${hour}` } | ||
| 367 | } | ||
| 368 | |||
| 369 | private generateResetPasswordKey (userId: number) { | ||
| 370 | return 'reset-password-' + userId | ||
| 371 | } | ||
| 372 | |||
| 373 | private generateTwoFactorRequestKey (userId: number, token: string) { | ||
| 374 | return 'two-factor-request-' + userId + '-' + token | ||
| 375 | } | ||
| 376 | |||
| 377 | private generateUserVerifyEmailKey (userId: number) { | ||
| 378 | return 'verify-email-user-' + userId | ||
| 379 | } | ||
| 380 | |||
| 381 | private generateRegistrationVerifyEmailKey (registrationId: number) { | ||
| 382 | return 'verify-email-registration-' + registrationId | ||
| 383 | } | ||
| 384 | |||
| 385 | private generateIPViewKey (ip: string, videoUUID: string) { | ||
| 386 | return `views-${videoUUID}-${ip}` | ||
| 387 | } | ||
| 388 | |||
| 389 | private generateContactFormKey (ip: string) { | ||
| 390 | return 'contact-form-' + ip | ||
| 391 | } | ||
| 392 | |||
| 393 | private generateAPUnavailabilityKey (url: string) { | ||
| 394 | return 'ap-unavailability-' + sha256(url) | ||
| 395 | } | ||
| 396 | |||
| 397 | /* ************ Redis helpers ************ */ | ||
| 398 | |||
| 399 | private getValue (key: string) { | ||
| 400 | return this.client.get(this.prefix + key) | ||
| 401 | } | ||
| 402 | |||
| 403 | private getSet (key: string) { | ||
| 404 | return this.client.smembers(this.prefix + key) | ||
| 405 | } | ||
| 406 | |||
| 407 | private addToSet (key: string, value: string) { | ||
| 408 | return this.client.sadd(this.prefix + key, value) | ||
| 409 | } | ||
| 410 | |||
| 411 | private deleteFromSet (key: string, value: string) { | ||
| 412 | return this.client.srem(this.prefix + key, value) | ||
| 413 | } | ||
| 414 | |||
| 415 | private deleteKey (key: string) { | ||
| 416 | return this.client.del(this.prefix + key) | ||
| 417 | } | ||
| 418 | |||
| 419 | private async getObject (key: string) { | ||
| 420 | const value = await this.getValue(key) | ||
| 421 | if (!value) return null | ||
| 422 | |||
| 423 | return JSON.parse(value) | ||
| 424 | } | ||
| 425 | |||
| 426 | private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) { | ||
| 427 | return this.setValue(key, JSON.stringify(value), expirationMilliseconds) | ||
| 428 | } | ||
| 429 | |||
| 430 | private async setValue (key: string, value: string, expirationMilliseconds?: number) { | ||
| 431 | const result = expirationMilliseconds !== undefined | ||
| 432 | ? await this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds) | ||
| 433 | : await this.client.set(this.prefix + key, value) | ||
| 434 | |||
| 435 | if (result !== 'OK') throw new Error('Redis set result is not OK.') | ||
| 436 | } | ||
| 437 | |||
| 438 | private removeValue (key: string) { | ||
| 439 | return this.client.del(this.prefix + key) | ||
| 440 | } | ||
| 441 | |||
| 442 | private increment (key: string) { | ||
| 443 | return this.client.incr(this.prefix + key) | ||
| 444 | } | ||
| 445 | |||
| 446 | private async exists (key: string) { | ||
| 447 | const result = await this.client.exists(this.prefix + key) | ||
| 448 | |||
| 449 | return result !== 0 | ||
| 450 | } | ||
| 451 | |||
| 452 | private setExpiration (key: string, ms: number) { | ||
| 453 | return this.client.expire(this.prefix + key, ms / 1000) | ||
| 454 | } | ||
| 455 | |||
| 456 | static get Instance () { | ||
| 457 | return this.instance || (this.instance = new this()) | ||
| 458 | } | ||
| 459 | } | ||
| 460 | |||
| 461 | // --------------------------------------------------------------------------- | ||
| 462 | |||
| 463 | export { | ||
| 464 | Redis | ||
| 465 | } | ||
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts deleted file mode 100644 index 2613d01be..000000000 --- a/server/lib/redundancy.ts +++ /dev/null | |||
| @@ -1,59 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 3 | import { CONFIG } from '@server/initializers/config' | ||
| 4 | import { ActorFollowModel } from '@server/models/actor/actor-follow' | ||
| 5 | import { getServerActor } from '@server/models/application/application' | ||
| 6 | import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models' | ||
| 7 | import { Activity } from '@shared/models' | ||
| 8 | import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' | ||
| 9 | import { sendUndoCacheFile } from './activitypub/send' | ||
| 10 | |||
| 11 | const lTags = loggerTagsFactory('redundancy') | ||
| 12 | |||
| 13 | async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { | ||
| 14 | const serverActor = await getServerActor() | ||
| 15 | |||
| 16 | // Local cache, send undo to remote instances | ||
| 17 | if (videoRedundancy.actorId === serverActor.id) await sendUndoCacheFile(serverActor, videoRedundancy, t) | ||
| 18 | |||
| 19 | await videoRedundancy.destroy({ transaction: t }) | ||
| 20 | } | ||
| 21 | |||
| 22 | async function removeRedundanciesOfServer (serverId: number) { | ||
| 23 | const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) | ||
| 24 | |||
| 25 | for (const redundancy of redundancies) { | ||
| 26 | await removeVideoRedundancy(redundancy) | ||
| 27 | } | ||
| 28 | } | ||
| 29 | |||
| 30 | async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) { | ||
| 31 | const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM | ||
| 32 | if (configAcceptFrom === 'nobody') { | ||
| 33 | logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id, lTags()) | ||
| 34 | return false | ||
| 35 | } | ||
| 36 | |||
| 37 | if (configAcceptFrom === 'followings') { | ||
| 38 | const serverActor = await getServerActor() | ||
| 39 | const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id) | ||
| 40 | |||
| 41 | if (allowed !== true) { | ||
| 42 | logger.info( | ||
| 43 | 'Do not accept remote redundancy %s because actor %s is not followed by our instance.', | ||
| 44 | activity.id, byActor.url, lTags() | ||
| 45 | ) | ||
| 46 | return false | ||
| 47 | } | ||
| 48 | } | ||
| 49 | |||
| 50 | return true | ||
| 51 | } | ||
| 52 | |||
| 53 | // --------------------------------------------------------------------------- | ||
| 54 | |||
| 55 | export { | ||
| 56 | isRedundancyAccepted, | ||
| 57 | removeRedundanciesOfServer, | ||
| 58 | removeVideoRedundancy | ||
| 59 | } | ||
diff --git a/server/lib/runners/index.ts b/server/lib/runners/index.ts deleted file mode 100644 index a737c7b59..000000000 --- a/server/lib/runners/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './job-handlers' | ||
| 2 | export * from './runner' | ||
| 3 | export * from './runner-urls' | ||
diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts deleted file mode 100644 index 329977de1..000000000 --- a/server/lib/runners/job-handlers/abstract-job-handler.ts +++ /dev/null | |||
| @@ -1,269 +0,0 @@ | |||
| 1 | import { throttle } from 'lodash' | ||
| 2 | import { saveInTransactionWithRetries } from '@server/helpers/database-utils' | ||
| 3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 4 | import { RUNNER_JOBS } from '@server/initializers/constants' | ||
| 5 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 6 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
| 7 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
| 8 | import { setAsUpdated } from '@server/models/shared' | ||
| 9 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 10 | import { pick } from '@shared/core-utils' | ||
| 11 | import { | ||
| 12 | RunnerJobLiveRTMPHLSTranscodingPayload, | ||
| 13 | RunnerJobLiveRTMPHLSTranscodingPrivatePayload, | ||
| 14 | RunnerJobState, | ||
| 15 | RunnerJobStudioTranscodingPayload, | ||
| 16 | RunnerJobSuccessPayload, | ||
| 17 | RunnerJobType, | ||
| 18 | RunnerJobUpdatePayload, | ||
| 19 | RunnerJobVideoStudioTranscodingPrivatePayload, | ||
| 20 | RunnerJobVODAudioMergeTranscodingPayload, | ||
| 21 | RunnerJobVODAudioMergeTranscodingPrivatePayload, | ||
| 22 | RunnerJobVODHLSTranscodingPayload, | ||
| 23 | RunnerJobVODHLSTranscodingPrivatePayload, | ||
| 24 | RunnerJobVODWebVideoTranscodingPayload, | ||
| 25 | RunnerJobVODWebVideoTranscodingPrivatePayload | ||
| 26 | } from '@shared/models' | ||
| 27 | |||
| 28 | type CreateRunnerJobArg = | ||
| 29 | { | ||
| 30 | type: Extract<RunnerJobType, 'vod-web-video-transcoding'> | ||
| 31 | payload: RunnerJobVODWebVideoTranscodingPayload | ||
| 32 | privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | ||
| 33 | } | | ||
| 34 | { | ||
| 35 | type: Extract<RunnerJobType, 'vod-hls-transcoding'> | ||
| 36 | payload: RunnerJobVODHLSTranscodingPayload | ||
| 37 | privatePayload: RunnerJobVODHLSTranscodingPrivatePayload | ||
| 38 | } | | ||
| 39 | { | ||
| 40 | type: Extract<RunnerJobType, 'vod-audio-merge-transcoding'> | ||
| 41 | payload: RunnerJobVODAudioMergeTranscodingPayload | ||
| 42 | privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload | ||
| 43 | } | | ||
| 44 | { | ||
| 45 | type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'> | ||
| 46 | payload: RunnerJobLiveRTMPHLSTranscodingPayload | ||
| 47 | privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload | ||
| 48 | } | | ||
| 49 | { | ||
| 50 | type: Extract<RunnerJobType, 'video-studio-transcoding'> | ||
| 51 | payload: RunnerJobStudioTranscodingPayload | ||
| 52 | privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload | ||
| 53 | } | ||
| 54 | |||
| 55 | export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> { | ||
| 56 | |||
| 57 | protected readonly lTags = loggerTagsFactory('runner') | ||
| 58 | |||
| 59 | static setJobAsUpdatedThrottled = throttle(setAsUpdated, 2000) | ||
| 60 | |||
| 61 | // --------------------------------------------------------------------------- | ||
| 62 | |||
| 63 | abstract create (options: C): Promise<MRunnerJob> | ||
| 64 | |||
| 65 | protected async createRunnerJob (options: CreateRunnerJobArg & { | ||
| 66 | jobUUID: string | ||
| 67 | priority: number | ||
| 68 | dependsOnRunnerJob?: MRunnerJob | ||
| 69 | }): Promise<MRunnerJob> { | ||
| 70 | const { priority, dependsOnRunnerJob } = options | ||
| 71 | |||
| 72 | logger.debug('Creating runner job', { options, ...this.lTags(options.type) }) | ||
| 73 | |||
| 74 | const runnerJob = new RunnerJobModel({ | ||
| 75 | ...pick(options, [ 'type', 'payload', 'privatePayload' ]), | ||
| 76 | |||
| 77 | uuid: options.jobUUID, | ||
| 78 | |||
| 79 | state: dependsOnRunnerJob | ||
| 80 | ? RunnerJobState.WAITING_FOR_PARENT_JOB | ||
| 81 | : RunnerJobState.PENDING, | ||
| 82 | |||
| 83 | dependsOnRunnerJobId: dependsOnRunnerJob?.id, | ||
| 84 | |||
| 85 | priority | ||
| 86 | }) | ||
| 87 | |||
| 88 | const job = await sequelizeTypescript.transaction(async transaction => { | ||
| 89 | return runnerJob.save({ transaction }) | ||
| 90 | }) | ||
| 91 | |||
| 92 | if (runnerJob.state === RunnerJobState.PENDING) { | ||
| 93 | PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() | ||
| 94 | } | ||
| 95 | |||
| 96 | return job | ||
| 97 | } | ||
| 98 | |||
| 99 | // --------------------------------------------------------------------------- | ||
| 100 | |||
| 101 | protected abstract specificUpdate (options: { | ||
| 102 | runnerJob: MRunnerJob | ||
| 103 | updatePayload?: U | ||
| 104 | }): Promise<void> | void | ||
| 105 | |||
| 106 | async update (options: { | ||
| 107 | runnerJob: MRunnerJob | ||
| 108 | progress?: number | ||
| 109 | updatePayload?: U | ||
| 110 | }) { | ||
| 111 | const { runnerJob, progress } = options | ||
| 112 | |||
| 113 | await this.specificUpdate(options) | ||
| 114 | |||
| 115 | if (progress) runnerJob.progress = progress | ||
| 116 | |||
| 117 | if (!runnerJob.changed()) { | ||
| 118 | try { | ||
| 119 | await AbstractJobHandler.setJobAsUpdatedThrottled({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id }) | ||
| 120 | } catch (err) { | ||
| 121 | logger.warn('Cannot set remote job as updated', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) | ||
| 122 | } | ||
| 123 | |||
| 124 | return | ||
| 125 | } | ||
| 126 | |||
| 127 | await saveInTransactionWithRetries(runnerJob) | ||
| 128 | } | ||
| 129 | |||
| 130 | // --------------------------------------------------------------------------- | ||
| 131 | |||
| 132 | async complete (options: { | ||
| 133 | runnerJob: MRunnerJob | ||
| 134 | resultPayload: S | ||
| 135 | }) { | ||
| 136 | const { runnerJob } = options | ||
| 137 | |||
| 138 | runnerJob.state = RunnerJobState.COMPLETING | ||
| 139 | await saveInTransactionWithRetries(runnerJob) | ||
| 140 | |||
| 141 | try { | ||
| 142 | await this.specificComplete(options) | ||
| 143 | |||
| 144 | runnerJob.state = RunnerJobState.COMPLETED | ||
| 145 | } catch (err) { | ||
| 146 | logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) | ||
| 147 | |||
| 148 | runnerJob.state = RunnerJobState.ERRORED | ||
| 149 | runnerJob.error = err.message | ||
| 150 | } | ||
| 151 | |||
| 152 | runnerJob.progress = null | ||
| 153 | runnerJob.finishedAt = new Date() | ||
| 154 | |||
| 155 | await saveInTransactionWithRetries(runnerJob) | ||
| 156 | |||
| 157 | const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob) | ||
| 158 | |||
| 159 | if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() | ||
| 160 | } | ||
| 161 | |||
| 162 | protected abstract specificComplete (options: { | ||
| 163 | runnerJob: MRunnerJob | ||
| 164 | resultPayload: S | ||
| 165 | }): Promise<void> | void | ||
| 166 | |||
| 167 | // --------------------------------------------------------------------------- | ||
| 168 | |||
| 169 | async cancel (options: { | ||
| 170 | runnerJob: MRunnerJob | ||
| 171 | fromParent?: boolean | ||
| 172 | }) { | ||
| 173 | const { runnerJob, fromParent } = options | ||
| 174 | |||
| 175 | await this.specificCancel(options) | ||
| 176 | |||
| 177 | const cancelState = fromParent | ||
| 178 | ? RunnerJobState.PARENT_CANCELLED | ||
| 179 | : RunnerJobState.CANCELLED | ||
| 180 | |||
| 181 | runnerJob.setToErrorOrCancel(cancelState) | ||
| 182 | |||
| 183 | await saveInTransactionWithRetries(runnerJob) | ||
| 184 | |||
| 185 | const children = await RunnerJobModel.listChildrenOf(runnerJob) | ||
| 186 | for (const child of children) { | ||
| 187 | logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid)) | ||
| 188 | |||
| 189 | await this.cancel({ runnerJob: child, fromParent: true }) | ||
| 190 | } | ||
| 191 | } | ||
| 192 | |||
| 193 | protected abstract specificCancel (options: { | ||
| 194 | runnerJob: MRunnerJob | ||
| 195 | }): Promise<void> | void | ||
| 196 | |||
| 197 | // --------------------------------------------------------------------------- | ||
| 198 | |||
| 199 | protected abstract isAbortSupported (): boolean | ||
| 200 | |||
| 201 | async abort (options: { | ||
| 202 | runnerJob: MRunnerJob | ||
| 203 | }) { | ||
| 204 | const { runnerJob } = options | ||
| 205 | |||
| 206 | if (this.isAbortSupported() !== true) { | ||
| 207 | return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' }) | ||
| 208 | } | ||
| 209 | |||
| 210 | await this.specificAbort(options) | ||
| 211 | |||
| 212 | runnerJob.resetToPending() | ||
| 213 | |||
| 214 | await saveInTransactionWithRetries(runnerJob) | ||
| 215 | } | ||
| 216 | |||
| 217 | protected setAbortState (runnerJob: MRunnerJob) { | ||
| 218 | runnerJob.resetToPending() | ||
| 219 | } | ||
| 220 | |||
| 221 | protected abstract specificAbort (options: { | ||
| 222 | runnerJob: MRunnerJob | ||
| 223 | }): Promise<void> | void | ||
| 224 | |||
| 225 | // --------------------------------------------------------------------------- | ||
| 226 | |||
| 227 | async error (options: { | ||
| 228 | runnerJob: MRunnerJob | ||
| 229 | message: string | ||
| 230 | fromParent?: boolean | ||
| 231 | }) { | ||
| 232 | const { runnerJob, message, fromParent } = options | ||
| 233 | |||
| 234 | const errorState = fromParent | ||
| 235 | ? RunnerJobState.PARENT_ERRORED | ||
| 236 | : RunnerJobState.ERRORED | ||
| 237 | |||
| 238 | const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES | ||
| 239 | ? RunnerJobState.PENDING | ||
| 240 | : errorState | ||
| 241 | |||
| 242 | await this.specificError({ ...options, nextState }) | ||
| 243 | |||
| 244 | if (nextState === errorState) { | ||
| 245 | runnerJob.setToErrorOrCancel(nextState) | ||
| 246 | runnerJob.error = message | ||
| 247 | } else { | ||
| 248 | runnerJob.resetToPending() | ||
| 249 | } | ||
| 250 | |||
| 251 | await saveInTransactionWithRetries(runnerJob) | ||
| 252 | |||
| 253 | if (runnerJob.state === errorState) { | ||
| 254 | const children = await RunnerJobModel.listChildrenOf(runnerJob) | ||
| 255 | |||
| 256 | for (const child of children) { | ||
| 257 | logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid)) | ||
| 258 | |||
| 259 | await this.error({ runnerJob: child, message: 'Parent error', fromParent: true }) | ||
| 260 | } | ||
| 261 | } | ||
| 262 | } | ||
| 263 | |||
| 264 | protected abstract specificError (options: { | ||
| 265 | runnerJob: MRunnerJob | ||
| 266 | message: string | ||
| 267 | nextState: RunnerJobState | ||
| 268 | }): Promise<void> | void | ||
| 269 | } | ||
diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts deleted file mode 100644 index f425828d9..000000000 --- a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts +++ /dev/null | |||
| @@ -1,66 +0,0 @@ | |||
| 1 | |||
| 2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' | ||
| 5 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 6 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 7 | import { RunnerJobState, RunnerJobSuccessPayload, RunnerJobUpdatePayload, RunnerJobVODPrivatePayload } from '@shared/models' | ||
| 8 | import { AbstractJobHandler } from './abstract-job-handler' | ||
| 9 | import { loadTranscodingRunnerVideo } from './shared' | ||
| 10 | |||
| 11 | // eslint-disable-next-line max-len | ||
| 12 | export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> { | ||
| 13 | |||
| 14 | protected isAbortSupported () { | ||
| 15 | return true | ||
| 16 | } | ||
| 17 | |||
| 18 | protected specificUpdate (_options: { | ||
| 19 | runnerJob: MRunnerJob | ||
| 20 | }) { | ||
| 21 | // empty | ||
| 22 | } | ||
| 23 | |||
| 24 | protected specificAbort (_options: { | ||
| 25 | runnerJob: MRunnerJob | ||
| 26 | }) { | ||
| 27 | // empty | ||
| 28 | } | ||
| 29 | |||
| 30 | protected async specificError (options: { | ||
| 31 | runnerJob: MRunnerJob | ||
| 32 | nextState: RunnerJobState | ||
| 33 | }) { | ||
| 34 | if (options.nextState !== RunnerJobState.ERRORED) return | ||
| 35 | |||
| 36 | const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) | ||
| 37 | if (!video) return | ||
| 38 | |||
| 39 | await moveToFailedTranscodingState(video) | ||
| 40 | |||
| 41 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
| 42 | } | ||
| 43 | |||
| 44 | protected async specificCancel (options: { | ||
| 45 | runnerJob: MRunnerJob | ||
| 46 | }) { | ||
| 47 | const { runnerJob } = options | ||
| 48 | |||
| 49 | const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) | ||
| 50 | if (!video) return | ||
| 51 | |||
| 52 | const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
| 53 | |||
| 54 | logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid)) | ||
| 55 | |||
| 56 | if (pending === 0) { | ||
| 57 | logger.info( | ||
| 58 | `All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`, | ||
| 59 | this.lTags(video.uuid) | ||
| 60 | ) | ||
| 61 | |||
| 62 | const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload | ||
| 63 | await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo }) | ||
| 64 | } | ||
| 65 | } | ||
| 66 | } | ||
diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts deleted file mode 100644 index 40ad2f97a..000000000 --- a/server/lib/runners/job-handlers/index.ts +++ /dev/null | |||
| @@ -1,7 +0,0 @@ | |||
| 1 | export * from './abstract-job-handler' | ||
| 2 | export * from './live-rtmp-hls-transcoding-job-handler' | ||
| 3 | export * from './runner-job-handlers' | ||
| 4 | export * from './video-studio-transcoding-job-handler' | ||
| 5 | export * from './vod-audio-merge-transcoding-job-handler' | ||
| 6 | export * from './vod-hls-transcoding-job-handler' | ||
| 7 | export * from './vod-web-video-transcoding-job-handler' | ||
diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts deleted file mode 100644 index 6b2894f8c..000000000 --- a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts +++ /dev/null | |||
| @@ -1,173 +0,0 @@ | |||
| 1 | import { move, remove } from 'fs-extra' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { JOB_PRIORITY } from '@server/initializers/constants' | ||
| 5 | import { LiveManager } from '@server/lib/live' | ||
| 6 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | ||
| 7 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 8 | import { buildUUID } from '@shared/extra-utils' | ||
| 9 | import { | ||
| 10 | LiveRTMPHLSTranscodingSuccess, | ||
| 11 | LiveRTMPHLSTranscodingUpdatePayload, | ||
| 12 | LiveVideoError, | ||
| 13 | RunnerJobLiveRTMPHLSTranscodingPayload, | ||
| 14 | RunnerJobLiveRTMPHLSTranscodingPrivatePayload, | ||
| 15 | RunnerJobState | ||
| 16 | } from '@shared/models' | ||
| 17 | import { AbstractJobHandler } from './abstract-job-handler' | ||
| 18 | |||
| 19 | type CreateOptions = { | ||
| 20 | video: MVideo | ||
| 21 | playlist: MStreamingPlaylist | ||
| 22 | |||
| 23 | sessionId: string | ||
| 24 | rtmpUrl: string | ||
| 25 | |||
| 26 | toTranscode: { | ||
| 27 | resolution: number | ||
| 28 | fps: number | ||
| 29 | }[] | ||
| 30 | |||
| 31 | segmentListSize: number | ||
| 32 | segmentDuration: number | ||
| 33 | |||
| 34 | outputDirectory: string | ||
| 35 | } | ||
| 36 | |||
| 37 | // eslint-disable-next-line max-len | ||
| 38 | export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateOptions, LiveRTMPHLSTranscodingUpdatePayload, LiveRTMPHLSTranscodingSuccess> { | ||
| 39 | |||
| 40 | async create (options: CreateOptions) { | ||
| 41 | const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory, sessionId } = options | ||
| 42 | |||
| 43 | const jobUUID = buildUUID() | ||
| 44 | const payload: RunnerJobLiveRTMPHLSTranscodingPayload = { | ||
| 45 | input: { | ||
| 46 | rtmpUrl | ||
| 47 | }, | ||
| 48 | output: { | ||
| 49 | toTranscode, | ||
| 50 | segmentListSize, | ||
| 51 | segmentDuration | ||
| 52 | } | ||
| 53 | } | ||
| 54 | |||
| 55 | const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = { | ||
| 56 | videoUUID: video.uuid, | ||
| 57 | masterPlaylistName: playlist.playlistFilename, | ||
| 58 | sessionId, | ||
| 59 | outputDirectory | ||
| 60 | } | ||
| 61 | |||
| 62 | const job = await this.createRunnerJob({ | ||
| 63 | type: 'live-rtmp-hls-transcoding', | ||
| 64 | jobUUID, | ||
| 65 | payload, | ||
| 66 | privatePayload, | ||
| 67 | priority: JOB_PRIORITY.TRANSCODING | ||
| 68 | }) | ||
| 69 | |||
| 70 | return job | ||
| 71 | } | ||
| 72 | |||
| 73 | // --------------------------------------------------------------------------- | ||
| 74 | |||
| 75 | protected async specificUpdate (options: { | ||
| 76 | runnerJob: MRunnerJob | ||
| 77 | updatePayload: LiveRTMPHLSTranscodingUpdatePayload | ||
| 78 | }) { | ||
| 79 | const { runnerJob, updatePayload } = options | ||
| 80 | |||
| 81 | const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload | ||
| 82 | const outputDirectory = privatePayload.outputDirectory | ||
| 83 | const videoUUID = privatePayload.videoUUID | ||
| 84 | |||
| 85 | // Always process the chunk first before moving m3u8 that references this chunk | ||
| 86 | if (updatePayload.type === 'add-chunk') { | ||
| 87 | await move( | ||
| 88 | updatePayload.videoChunkFile as string, | ||
| 89 | join(outputDirectory, updatePayload.videoChunkFilename), | ||
| 90 | { overwrite: true } | ||
| 91 | ) | ||
| 92 | } else if (updatePayload.type === 'remove-chunk') { | ||
| 93 | await remove(join(outputDirectory, updatePayload.videoChunkFilename)) | ||
| 94 | } | ||
| 95 | |||
| 96 | if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) { | ||
| 97 | await move( | ||
| 98 | updatePayload.resolutionPlaylistFile as string, | ||
| 99 | join(outputDirectory, updatePayload.resolutionPlaylistFilename), | ||
| 100 | { overwrite: true } | ||
| 101 | ) | ||
| 102 | } | ||
| 103 | |||
| 104 | if (updatePayload.masterPlaylistFile) { | ||
| 105 | await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true }) | ||
| 106 | } | ||
| 107 | |||
| 108 | logger.debug( | ||
| 109 | 'Runner live RTMP to HLS job %s for %s updated.', | ||
| 110 | runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) } | ||
| 111 | ) | ||
| 112 | } | ||
| 113 | |||
| 114 | // --------------------------------------------------------------------------- | ||
| 115 | |||
| 116 | protected specificComplete (options: { | ||
| 117 | runnerJob: MRunnerJob | ||
| 118 | }) { | ||
| 119 | return this.stopLive({ | ||
| 120 | runnerJob: options.runnerJob, | ||
| 121 | type: 'ended' | ||
| 122 | }) | ||
| 123 | } | ||
| 124 | |||
| 125 | // --------------------------------------------------------------------------- | ||
| 126 | |||
| 127 | protected isAbortSupported () { | ||
| 128 | return false | ||
| 129 | } | ||
| 130 | |||
| 131 | protected specificAbort () { | ||
| 132 | throw new Error('Not implemented') | ||
| 133 | } | ||
| 134 | |||
| 135 | protected specificError (options: { | ||
| 136 | runnerJob: MRunnerJob | ||
| 137 | nextState: RunnerJobState | ||
| 138 | }) { | ||
| 139 | return this.stopLive({ | ||
| 140 | runnerJob: options.runnerJob, | ||
| 141 | type: 'errored' | ||
| 142 | }) | ||
| 143 | } | ||
| 144 | |||
| 145 | protected specificCancel (options: { | ||
| 146 | runnerJob: MRunnerJob | ||
| 147 | }) { | ||
| 148 | return this.stopLive({ | ||
| 149 | runnerJob: options.runnerJob, | ||
| 150 | type: 'cancelled' | ||
| 151 | }) | ||
| 152 | } | ||
| 153 | |||
| 154 | private stopLive (options: { | ||
| 155 | runnerJob: MRunnerJob | ||
| 156 | type: 'ended' | 'errored' | 'cancelled' | ||
| 157 | }) { | ||
| 158 | const { runnerJob, type } = options | ||
| 159 | |||
| 160 | const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload | ||
| 161 | const videoUUID = privatePayload.videoUUID | ||
| 162 | |||
| 163 | const errorType = { | ||
| 164 | ended: null, | ||
| 165 | errored: LiveVideoError.RUNNER_JOB_ERROR, | ||
| 166 | cancelled: LiveVideoError.RUNNER_JOB_CANCEL | ||
| 167 | } | ||
| 168 | |||
| 169 | LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type]) | ||
| 170 | |||
| 171 | logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID)) | ||
| 172 | } | ||
| 173 | } | ||
diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts deleted file mode 100644 index 85551c365..000000000 --- a/server/lib/runners/job-handlers/runner-job-handlers.ts +++ /dev/null | |||
| @@ -1,20 +0,0 @@ | |||
| 1 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 2 | import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' | ||
| 3 | import { AbstractJobHandler } from './abstract-job-handler' | ||
| 4 | import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' | ||
| 5 | import { VideoStudioTranscodingJobHandler } from './video-studio-transcoding-job-handler' | ||
| 6 | import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' | ||
| 7 | import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' | ||
| 8 | import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' | ||
| 9 | |||
| 10 | const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, RunnerJobUpdatePayload, RunnerJobSuccessPayload>> = { | ||
| 11 | 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, | ||
| 12 | 'vod-hls-transcoding': VODHLSTranscodingJobHandler, | ||
| 13 | 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, | ||
| 14 | 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler, | ||
| 15 | 'video-studio-transcoding': VideoStudioTranscodingJobHandler | ||
| 16 | } | ||
| 17 | |||
| 18 | export function getRunnerJobHandlerClass (job: MRunnerJob) { | ||
| 19 | return processors[job.type] | ||
| 20 | } | ||
diff --git a/server/lib/runners/job-handlers/shared/index.ts b/server/lib/runners/job-handlers/shared/index.ts deleted file mode 100644 index 348273ae2..000000000 --- a/server/lib/runners/job-handlers/shared/index.ts +++ /dev/null | |||
| @@ -1 +0,0 @@ | |||
| 1 | export * from './vod-helpers' | ||
diff --git a/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/lib/runners/job-handlers/shared/vod-helpers.ts deleted file mode 100644 index 1a2ad02ca..000000000 --- a/server/lib/runners/job-handlers/shared/vod-helpers.ts +++ /dev/null | |||
| @@ -1,44 +0,0 @@ | |||
| 1 | import { move } from 'fs-extra' | ||
| 2 | import { dirname, join } from 'path' | ||
| 3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' | ||
| 4 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | ||
| 5 | import { onWebVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding' | ||
| 6 | import { buildNewFile } from '@server/lib/video-file' | ||
| 7 | import { VideoModel } from '@server/models/video/video' | ||
| 8 | import { MVideoFullLight } from '@server/types/models' | ||
| 9 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 10 | import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@shared/models' | ||
| 11 | |||
| 12 | export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { | ||
| 13 | video: MVideoFullLight | ||
| 14 | videoFilePath: string | ||
| 15 | privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload | ||
| 16 | }) { | ||
| 17 | const { video, videoFilePath, privatePayload } = options | ||
| 18 | |||
| 19 | const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' }) | ||
| 20 | videoFile.videoId = video.id | ||
| 21 | |||
| 22 | const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) | ||
| 23 | await move(videoFilePath, newVideoFilePath) | ||
| 24 | |||
| 25 | await onWebVideoFileTranscoding({ | ||
| 26 | video, | ||
| 27 | videoFile, | ||
| 28 | videoOutputPath: newVideoFilePath | ||
| 29 | }) | ||
| 30 | |||
| 31 | await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) | ||
| 32 | } | ||
| 33 | |||
| 34 | export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) { | ||
| 35 | const videoUUID = runnerJob.privatePayload.videoUUID | ||
| 36 | |||
| 37 | const video = await VideoModel.loadFull(videoUUID) | ||
| 38 | if (!video) { | ||
| 39 | logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID)) | ||
| 40 | return undefined | ||
| 41 | } | ||
| 42 | |||
| 43 | return video | ||
| 44 | } | ||
diff --git a/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts b/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts deleted file mode 100644 index f604382b7..000000000 --- a/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts +++ /dev/null | |||
| @@ -1,157 +0,0 @@ | |||
| 1 | |||
| 2 | import { basename } from 'path' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' | ||
| 5 | import { MVideo } from '@server/types/models' | ||
| 6 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 7 | import { buildUUID } from '@shared/extra-utils' | ||
| 8 | import { | ||
| 9 | isVideoStudioTaskIntro, | ||
| 10 | isVideoStudioTaskOutro, | ||
| 11 | isVideoStudioTaskWatermark, | ||
| 12 | RunnerJobState, | ||
| 13 | RunnerJobUpdatePayload, | ||
| 14 | RunnerJobStudioTranscodingPayload, | ||
| 15 | RunnerJobVideoStudioTranscodingPrivatePayload, | ||
| 16 | VideoStudioTranscodingSuccess, | ||
| 17 | VideoState, | ||
| 18 | VideoStudioTaskPayload | ||
| 19 | } from '@shared/models' | ||
| 20 | import { generateRunnerEditionTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' | ||
| 21 | import { AbstractJobHandler } from './abstract-job-handler' | ||
| 22 | import { loadTranscodingRunnerVideo } from './shared' | ||
| 23 | |||
| 24 | type CreateOptions = { | ||
| 25 | video: MVideo | ||
| 26 | tasks: VideoStudioTaskPayload[] | ||
| 27 | priority: number | ||
| 28 | } | ||
| 29 | |||
| 30 | // eslint-disable-next-line max-len | ||
| 31 | export class VideoStudioTranscodingJobHandler extends AbstractJobHandler<CreateOptions, RunnerJobUpdatePayload, VideoStudioTranscodingSuccess> { | ||
| 32 | |||
| 33 | async create (options: CreateOptions) { | ||
| 34 | const { video, priority, tasks } = options | ||
| 35 | |||
| 36 | const jobUUID = buildUUID() | ||
| 37 | const payload: RunnerJobStudioTranscodingPayload = { | ||
| 38 | input: { | ||
| 39 | videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) | ||
| 40 | }, | ||
| 41 | tasks: tasks.map(t => { | ||
| 42 | if (isVideoStudioTaskIntro(t) || isVideoStudioTaskOutro(t)) { | ||
| 43 | return { | ||
| 44 | ...t, | ||
| 45 | |||
| 46 | options: { | ||
| 47 | ...t.options, | ||
| 48 | |||
| 49 | file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) | ||
| 50 | } | ||
| 51 | } | ||
| 52 | } | ||
| 53 | |||
| 54 | if (isVideoStudioTaskWatermark(t)) { | ||
| 55 | return { | ||
| 56 | ...t, | ||
| 57 | |||
| 58 | options: { | ||
| 59 | ...t.options, | ||
| 60 | |||
| 61 | file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) | ||
| 62 | } | ||
| 63 | } | ||
| 64 | } | ||
| 65 | |||
| 66 | return t | ||
| 67 | }) | ||
| 68 | } | ||
| 69 | |||
| 70 | const privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload = { | ||
| 71 | videoUUID: video.uuid, | ||
| 72 | originalTasks: tasks | ||
| 73 | } | ||
| 74 | |||
| 75 | const job = await this.createRunnerJob({ | ||
| 76 | type: 'video-studio-transcoding', | ||
| 77 | jobUUID, | ||
| 78 | payload, | ||
| 79 | privatePayload, | ||
| 80 | priority | ||
| 81 | }) | ||
| 82 | |||
| 83 | return job | ||
| 84 | } | ||
| 85 | |||
| 86 | // --------------------------------------------------------------------------- | ||
| 87 | |||
| 88 | protected isAbortSupported () { | ||
| 89 | return true | ||
| 90 | } | ||
| 91 | |||
| 92 | protected specificUpdate (_options: { | ||
| 93 | runnerJob: MRunnerJob | ||
| 94 | }) { | ||
| 95 | // empty | ||
| 96 | } | ||
| 97 | |||
| 98 | protected specificAbort (_options: { | ||
| 99 | runnerJob: MRunnerJob | ||
| 100 | }) { | ||
| 101 | // empty | ||
| 102 | } | ||
| 103 | |||
| 104 | protected async specificComplete (options: { | ||
| 105 | runnerJob: MRunnerJob | ||
| 106 | resultPayload: VideoStudioTranscodingSuccess | ||
| 107 | }) { | ||
| 108 | const { runnerJob, resultPayload } = options | ||
| 109 | const privatePayload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload | ||
| 110 | |||
| 111 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
| 112 | if (!video) { | ||
| 113 | await safeCleanupStudioTMPFiles(privatePayload.originalTasks) | ||
| 114 | |||
| 115 | } | ||
| 116 | |||
| 117 | const videoFilePath = resultPayload.videoFile as string | ||
| 118 | |||
| 119 | await onVideoStudioEnded({ video, editionResultPath: videoFilePath, tasks: privatePayload.originalTasks }) | ||
| 120 | |||
| 121 | logger.info( | ||
| 122 | 'Runner video edition transcoding job %s for %s ended.', | ||
| 123 | runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) | ||
| 124 | ) | ||
| 125 | } | ||
| 126 | |||
| 127 | protected specificError (options: { | ||
| 128 | runnerJob: MRunnerJob | ||
| 129 | nextState: RunnerJobState | ||
| 130 | }) { | ||
| 131 | if (options.nextState === RunnerJobState.ERRORED) { | ||
| 132 | return this.specificErrorOrCancel(options) | ||
| 133 | } | ||
| 134 | |||
| 135 | return Promise.resolve() | ||
| 136 | } | ||
| 137 | |||
| 138 | protected specificCancel (options: { | ||
| 139 | runnerJob: MRunnerJob | ||
| 140 | }) { | ||
| 141 | return this.specificErrorOrCancel(options) | ||
| 142 | } | ||
| 143 | |||
| 144 | private async specificErrorOrCancel (options: { | ||
| 145 | runnerJob: MRunnerJob | ||
| 146 | }) { | ||
| 147 | const { runnerJob } = options | ||
| 148 | |||
| 149 | const payload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload | ||
| 150 | await safeCleanupStudioTMPFiles(payload.originalTasks) | ||
| 151 | |||
| 152 | const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) | ||
| 153 | if (!video) return | ||
| 154 | |||
| 155 | return video.setNewState(VideoState.PUBLISHED, false, undefined) | ||
| 156 | } | ||
| 157 | } | ||
diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts deleted file mode 100644 index 137a94535..000000000 --- a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts +++ /dev/null | |||
| @@ -1,97 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 3 | import { MVideo } from '@server/types/models' | ||
| 4 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 5 | import { pick } from '@shared/core-utils' | ||
| 6 | import { buildUUID } from '@shared/extra-utils' | ||
| 7 | import { getVideoStreamDuration } from '@shared/ffmpeg' | ||
| 8 | import { | ||
| 9 | RunnerJobUpdatePayload, | ||
| 10 | RunnerJobVODAudioMergeTranscodingPayload, | ||
| 11 | RunnerJobVODWebVideoTranscodingPrivatePayload, | ||
| 12 | VODAudioMergeTranscodingSuccess | ||
| 13 | } from '@shared/models' | ||
| 14 | import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls' | ||
| 15 | import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' | ||
| 16 | import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' | ||
| 17 | |||
| 18 | type CreateOptions = { | ||
| 19 | video: MVideo | ||
| 20 | isNewVideo: boolean | ||
| 21 | resolution: number | ||
| 22 | fps: number | ||
| 23 | priority: number | ||
| 24 | dependsOnRunnerJob?: MRunnerJob | ||
| 25 | } | ||
| 26 | |||
| 27 | // eslint-disable-next-line max-len | ||
| 28 | export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODAudioMergeTranscodingSuccess> { | ||
| 29 | |||
| 30 | async create (options: CreateOptions) { | ||
| 31 | const { video, resolution, fps, priority, dependsOnRunnerJob } = options | ||
| 32 | |||
| 33 | const jobUUID = buildUUID() | ||
| 34 | const payload: RunnerJobVODAudioMergeTranscodingPayload = { | ||
| 35 | input: { | ||
| 36 | audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid), | ||
| 37 | previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid) | ||
| 38 | }, | ||
| 39 | output: { | ||
| 40 | resolution, | ||
| 41 | fps | ||
| 42 | } | ||
| 43 | } | ||
| 44 | |||
| 45 | const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { | ||
| 46 | ...pick(options, [ 'isNewVideo' ]), | ||
| 47 | |||
| 48 | videoUUID: video.uuid | ||
| 49 | } | ||
| 50 | |||
| 51 | const job = await this.createRunnerJob({ | ||
| 52 | type: 'vod-audio-merge-transcoding', | ||
| 53 | jobUUID, | ||
| 54 | payload, | ||
| 55 | privatePayload, | ||
| 56 | priority, | ||
| 57 | dependsOnRunnerJob | ||
| 58 | }) | ||
| 59 | |||
| 60 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
| 61 | |||
| 62 | return job | ||
| 63 | } | ||
| 64 | |||
| 65 | // --------------------------------------------------------------------------- | ||
| 66 | |||
| 67 | protected async specificComplete (options: { | ||
| 68 | runnerJob: MRunnerJob | ||
| 69 | resultPayload: VODAudioMergeTranscodingSuccess | ||
| 70 | }) { | ||
| 71 | const { runnerJob, resultPayload } = options | ||
| 72 | const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload | ||
| 73 | |||
| 74 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
| 75 | if (!video) return | ||
| 76 | |||
| 77 | const videoFilePath = resultPayload.videoFile as string | ||
| 78 | |||
| 79 | // ffmpeg generated a new video file, so update the video duration | ||
| 80 | // See https://trac.ffmpeg.org/ticket/5456 | ||
| 81 | video.duration = await getVideoStreamDuration(videoFilePath) | ||
| 82 | await video.save() | ||
| 83 | |||
| 84 | // We can remove the old audio file | ||
| 85 | const oldAudioFile = video.VideoFiles[0] | ||
| 86 | await video.removeWebVideoFile(oldAudioFile) | ||
| 87 | await oldAudioFile.destroy() | ||
| 88 | video.VideoFiles = [] | ||
| 89 | |||
| 90 | await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) | ||
| 91 | |||
| 92 | logger.info( | ||
| 93 | 'Runner VOD audio merge transcoding job %s for %s ended.', | ||
| 94 | runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) | ||
| 95 | ) | ||
| 96 | } | ||
| 97 | } | ||
diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts deleted file mode 100644 index 02845952c..000000000 --- a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts +++ /dev/null | |||
| @@ -1,114 +0,0 @@ | |||
| 1 | import { move } from 'fs-extra' | ||
| 2 | import { dirname, join } from 'path' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { renameVideoFileInPlaylist } from '@server/lib/hls' | ||
| 5 | import { getHlsResolutionPlaylistFilename } from '@server/lib/paths' | ||
| 6 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | ||
| 7 | import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding' | ||
| 8 | import { buildNewFile, removeAllWebVideoFiles } from '@server/lib/video-file' | ||
| 9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 10 | import { MVideo } from '@server/types/models' | ||
| 11 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 12 | import { pick } from '@shared/core-utils' | ||
| 13 | import { buildUUID } from '@shared/extra-utils' | ||
| 14 | import { | ||
| 15 | RunnerJobUpdatePayload, | ||
| 16 | RunnerJobVODHLSTranscodingPayload, | ||
| 17 | RunnerJobVODHLSTranscodingPrivatePayload, | ||
| 18 | VODHLSTranscodingSuccess | ||
| 19 | } from '@shared/models' | ||
| 20 | import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' | ||
| 21 | import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' | ||
| 22 | import { loadTranscodingRunnerVideo } from './shared' | ||
| 23 | |||
| 24 | type CreateOptions = { | ||
| 25 | video: MVideo | ||
| 26 | isNewVideo: boolean | ||
| 27 | deleteWebVideoFiles: boolean | ||
| 28 | resolution: number | ||
| 29 | fps: number | ||
| 30 | priority: number | ||
| 31 | dependsOnRunnerJob?: MRunnerJob | ||
| 32 | } | ||
| 33 | |||
| 34 | // eslint-disable-next-line max-len | ||
| 35 | export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODHLSTranscodingSuccess> { | ||
| 36 | |||
| 37 | async create (options: CreateOptions) { | ||
| 38 | const { video, resolution, fps, dependsOnRunnerJob, priority } = options | ||
| 39 | |||
| 40 | const jobUUID = buildUUID() | ||
| 41 | |||
| 42 | const payload: RunnerJobVODHLSTranscodingPayload = { | ||
| 43 | input: { | ||
| 44 | videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) | ||
| 45 | }, | ||
| 46 | output: { | ||
| 47 | resolution, | ||
| 48 | fps | ||
| 49 | } | ||
| 50 | } | ||
| 51 | |||
| 52 | const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = { | ||
| 53 | ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]), | ||
| 54 | |||
| 55 | videoUUID: video.uuid | ||
| 56 | } | ||
| 57 | |||
| 58 | const job = await this.createRunnerJob({ | ||
| 59 | type: 'vod-hls-transcoding', | ||
| 60 | jobUUID, | ||
| 61 | payload, | ||
| 62 | privatePayload, | ||
| 63 | priority, | ||
| 64 | dependsOnRunnerJob | ||
| 65 | }) | ||
| 66 | |||
| 67 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
| 68 | |||
| 69 | return job | ||
| 70 | } | ||
| 71 | |||
| 72 | // --------------------------------------------------------------------------- | ||
| 73 | |||
| 74 | protected async specificComplete (options: { | ||
| 75 | runnerJob: MRunnerJob | ||
| 76 | resultPayload: VODHLSTranscodingSuccess | ||
| 77 | }) { | ||
| 78 | const { runnerJob, resultPayload } = options | ||
| 79 | const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload | ||
| 80 | |||
| 81 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
| 82 | if (!video) return | ||
| 83 | |||
| 84 | const videoFilePath = resultPayload.videoFile as string | ||
| 85 | const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string | ||
| 86 | |||
| 87 | const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' }) | ||
| 88 | const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) | ||
| 89 | await move(videoFilePath, newVideoFilePath) | ||
| 90 | |||
| 91 | const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename) | ||
| 92 | const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename) | ||
| 93 | await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath) | ||
| 94 | |||
| 95 | await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename) | ||
| 96 | |||
| 97 | await onHLSVideoFileTranscoding({ | ||
| 98 | video, | ||
| 99 | videoFile, | ||
| 100 | m3u8OutputPath: newResolutionPlaylistFilePath, | ||
| 101 | videoOutputPath: newVideoFilePath | ||
| 102 | }) | ||
| 103 | |||
| 104 | await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) | ||
| 105 | |||
| 106 | if (privatePayload.deleteWebVideoFiles === true) { | ||
| 107 | logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) | ||
| 108 | |||
| 109 | await removeAllWebVideoFiles(video) | ||
| 110 | } | ||
| 111 | |||
| 112 | logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) | ||
| 113 | } | ||
| 114 | } | ||
diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts deleted file mode 100644 index 9ee8ab88e..000000000 --- a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts +++ /dev/null | |||
| @@ -1,84 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 3 | import { MVideo } from '@server/types/models' | ||
| 4 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 5 | import { pick } from '@shared/core-utils' | ||
| 6 | import { buildUUID } from '@shared/extra-utils' | ||
| 7 | import { | ||
| 8 | RunnerJobUpdatePayload, | ||
| 9 | RunnerJobVODWebVideoTranscodingPayload, | ||
| 10 | RunnerJobVODWebVideoTranscodingPrivatePayload, | ||
| 11 | VODWebVideoTranscodingSuccess | ||
| 12 | } from '@shared/models' | ||
| 13 | import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' | ||
| 14 | import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' | ||
| 15 | import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' | ||
| 16 | |||
| 17 | type CreateOptions = { | ||
| 18 | video: MVideo | ||
| 19 | isNewVideo: boolean | ||
| 20 | resolution: number | ||
| 21 | fps: number | ||
| 22 | priority: number | ||
| 23 | dependsOnRunnerJob?: MRunnerJob | ||
| 24 | } | ||
| 25 | |||
| 26 | // eslint-disable-next-line max-len | ||
| 27 | export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODWebVideoTranscodingSuccess> { | ||
| 28 | |||
| 29 | async create (options: CreateOptions) { | ||
| 30 | const { video, resolution, fps, priority, dependsOnRunnerJob } = options | ||
| 31 | |||
| 32 | const jobUUID = buildUUID() | ||
| 33 | const payload: RunnerJobVODWebVideoTranscodingPayload = { | ||
| 34 | input: { | ||
| 35 | videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) | ||
| 36 | }, | ||
| 37 | output: { | ||
| 38 | resolution, | ||
| 39 | fps | ||
| 40 | } | ||
| 41 | } | ||
| 42 | |||
| 43 | const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { | ||
| 44 | ...pick(options, [ 'isNewVideo' ]), | ||
| 45 | |||
| 46 | videoUUID: video.uuid | ||
| 47 | } | ||
| 48 | |||
| 49 | const job = await this.createRunnerJob({ | ||
| 50 | type: 'vod-web-video-transcoding', | ||
| 51 | jobUUID, | ||
| 52 | payload, | ||
| 53 | privatePayload, | ||
| 54 | dependsOnRunnerJob, | ||
| 55 | priority | ||
| 56 | }) | ||
| 57 | |||
| 58 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
| 59 | |||
| 60 | return job | ||
| 61 | } | ||
| 62 | |||
| 63 | // --------------------------------------------------------------------------- | ||
| 64 | |||
| 65 | protected async specificComplete (options: { | ||
| 66 | runnerJob: MRunnerJob | ||
| 67 | resultPayload: VODWebVideoTranscodingSuccess | ||
| 68 | }) { | ||
| 69 | const { runnerJob, resultPayload } = options | ||
| 70 | const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload | ||
| 71 | |||
| 72 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
| 73 | if (!video) return | ||
| 74 | |||
| 75 | const videoFilePath = resultPayload.videoFile as string | ||
| 76 | |||
| 77 | await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) | ||
| 78 | |||
| 79 | logger.info( | ||
| 80 | 'Runner VOD web video transcoding job %s for %s ended.', | ||
| 81 | runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) | ||
| 82 | ) | ||
| 83 | } | ||
| 84 | } | ||
diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts deleted file mode 100644 index a27060b33..000000000 --- a/server/lib/runners/runner-urls.ts +++ /dev/null | |||
| @@ -1,13 +0,0 @@ | |||
| 1 | import { WEBSERVER } from '@server/initializers/constants' | ||
| 2 | |||
| 3 | export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) { | ||
| 4 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality' | ||
| 5 | } | ||
| 6 | |||
| 7 | export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { | ||
| 8 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' | ||
| 9 | } | ||
| 10 | |||
| 11 | export function generateRunnerEditionTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string, filename: string) { | ||
| 12 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/studio/task-files/' + filename | ||
| 13 | } | ||
diff --git a/server/lib/runners/runner.ts b/server/lib/runners/runner.ts deleted file mode 100644 index 947fdb3f0..000000000 --- a/server/lib/runners/runner.ts +++ /dev/null | |||
| @@ -1,49 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 5 | import { MRunner, MRunnerJob } from '@server/types/models/runners' | ||
| 6 | import { RUNNER_JOBS } from '@server/initializers/constants' | ||
| 7 | import { RunnerJobState } from '@shared/models' | ||
| 8 | |||
| 9 | const lTags = loggerTagsFactory('runner') | ||
| 10 | |||
| 11 | const updatingRunner = new Set<number>() | ||
| 12 | |||
| 13 | function updateLastRunnerContact (req: express.Request, runner: MRunner) { | ||
| 14 | const now = new Date() | ||
| 15 | |||
| 16 | // Don't update last runner contact too often | ||
| 17 | if (now.getTime() - runner.lastContact.getTime() < RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL) return | ||
| 18 | if (updatingRunner.has(runner.id)) return | ||
| 19 | |||
| 20 | updatingRunner.add(runner.id) | ||
| 21 | |||
| 22 | runner.lastContact = now | ||
| 23 | runner.ip = req.ip | ||
| 24 | |||
| 25 | logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name)) | ||
| 26 | |||
| 27 | retryTransactionWrapper(() => { | ||
| 28 | return sequelizeTypescript.transaction(async transaction => { | ||
| 29 | return runner.save({ transaction }) | ||
| 30 | }) | ||
| 31 | }) | ||
| 32 | .catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) })) | ||
| 33 | .finally(() => updatingRunner.delete(runner.id)) | ||
| 34 | } | ||
| 35 | |||
| 36 | function runnerJobCanBeCancelled (runnerJob: MRunnerJob) { | ||
| 37 | const allowedStates = new Set<RunnerJobState>([ | ||
| 38 | RunnerJobState.PENDING, | ||
| 39 | RunnerJobState.PROCESSING, | ||
| 40 | RunnerJobState.WAITING_FOR_PARENT_JOB | ||
| 41 | ]) | ||
| 42 | |||
| 43 | return allowedStates.has(runnerJob.state) | ||
| 44 | } | ||
| 45 | |||
| 46 | export { | ||
| 47 | updateLastRunnerContact, | ||
| 48 | runnerJobCanBeCancelled | ||
| 49 | } | ||
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts deleted file mode 100644 index f3d51a22e..000000000 --- a/server/lib/schedulers/abstract-scheduler.ts +++ /dev/null | |||
| @@ -1,35 +0,0 @@ | |||
| 1 | import Bluebird from 'bluebird' | ||
| 2 | import { logger } from '../../helpers/logger' | ||
| 3 | |||
| 4 | export abstract class AbstractScheduler { | ||
| 5 | |||
| 6 | protected abstract schedulerIntervalMs: number | ||
| 7 | |||
| 8 | private interval: NodeJS.Timer | ||
| 9 | private isRunning = false | ||
| 10 | |||
| 11 | enable () { | ||
| 12 | if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') | ||
| 13 | |||
| 14 | this.interval = setInterval(() => this.execute(), this.schedulerIntervalMs) | ||
| 15 | } | ||
| 16 | |||
| 17 | disable () { | ||
| 18 | clearInterval(this.interval) | ||
| 19 | } | ||
| 20 | |||
| 21 | async execute () { | ||
| 22 | if (this.isRunning === true) return | ||
| 23 | this.isRunning = true | ||
| 24 | |||
| 25 | try { | ||
| 26 | await this.internalExecute() | ||
| 27 | } catch (err) { | ||
| 28 | logger.error('Cannot execute %s scheduler.', this.constructor.name, { err }) | ||
| 29 | } finally { | ||
| 30 | this.isRunning = false | ||
| 31 | } | ||
| 32 | } | ||
| 33 | |||
| 34 | protected abstract internalExecute (): Promise<any> | Bluebird<any> | ||
| 35 | } | ||
diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts deleted file mode 100644 index e1c56c135..000000000 --- a/server/lib/schedulers/actor-follow-scheduler.ts +++ /dev/null | |||
| @@ -1,54 +0,0 @@ | |||
| 1 | import { isTestOrDevInstance } from '../../helpers/core-utils' | ||
| 2 | import { logger } from '../../helpers/logger' | ||
| 3 | import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 4 | import { ActorFollowModel } from '../../models/actor/actor-follow' | ||
| 5 | import { ActorFollowHealthCache } from '../actor-follow-health-cache' | ||
| 6 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 7 | |||
| 8 | export class ActorFollowScheduler extends AbstractScheduler { | ||
| 9 | |||
| 10 | private static instance: AbstractScheduler | ||
| 11 | |||
| 12 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES | ||
| 13 | |||
| 14 | private constructor () { | ||
| 15 | super() | ||
| 16 | } | ||
| 17 | |||
| 18 | protected async internalExecute () { | ||
| 19 | await this.processPendingScores() | ||
| 20 | |||
| 21 | await this.removeBadActorFollows() | ||
| 22 | } | ||
| 23 | |||
| 24 | private async processPendingScores () { | ||
| 25 | const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore() | ||
| 26 | const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds() | ||
| 27 | const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds() | ||
| 28 | |||
| 29 | ActorFollowHealthCache.Instance.clearPendingFollowsScore() | ||
| 30 | ActorFollowHealthCache.Instance.clearBadFollowingServerIds() | ||
| 31 | ActorFollowHealthCache.Instance.clearGoodFollowingServerIds() | ||
| 32 | |||
| 33 | for (const inbox of Object.keys(pendingScores)) { | ||
| 34 | await ActorFollowModel.updateScore(inbox, pendingScores[inbox]) | ||
| 35 | } | ||
| 36 | |||
| 37 | await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY) | ||
| 38 | await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS) | ||
| 39 | } | ||
| 40 | |||
| 41 | private async removeBadActorFollows () { | ||
| 42 | if (!isTestOrDevInstance()) logger.info('Removing bad actor follows (scheduler).') | ||
| 43 | |||
| 44 | try { | ||
| 45 | await ActorFollowModel.removeBadActorFollows() | ||
| 46 | } catch (err) { | ||
| 47 | logger.error('Error in bad actor follows scheduler.', { err }) | ||
| 48 | } | ||
| 49 | } | ||
| 50 | |||
| 51 | static get Instance () { | ||
| 52 | return this.instance || (this.instance = new this()) | ||
| 53 | } | ||
| 54 | } | ||
diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts deleted file mode 100644 index 956ece749..000000000 --- a/server/lib/schedulers/auto-follow-index-instances.ts +++ /dev/null | |||
| @@ -1,75 +0,0 @@ | |||
| 1 | import { chunk } from 'lodash' | ||
| 2 | import { doJSONRequest } from '@server/helpers/requests' | ||
| 3 | import { JobQueue } from '@server/lib/job-queue' | ||
| 4 | import { ActorFollowModel } from '@server/models/actor/actor-follow' | ||
| 5 | import { getServerActor } from '@server/models/application/application' | ||
| 6 | import { logger } from '../../helpers/logger' | ||
| 7 | import { CONFIG } from '../../initializers/config' | ||
| 8 | import { SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants' | ||
| 9 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 10 | |||
| 11 | export class AutoFollowIndexInstances extends AbstractScheduler { | ||
| 12 | |||
| 13 | private static instance: AbstractScheduler | ||
| 14 | |||
| 15 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES | ||
| 16 | |||
| 17 | private lastCheck: Date | ||
| 18 | |||
| 19 | private constructor () { | ||
| 20 | super() | ||
| 21 | } | ||
| 22 | |||
| 23 | protected async internalExecute () { | ||
| 24 | return this.autoFollow() | ||
| 25 | } | ||
| 26 | |||
| 27 | private async autoFollow () { | ||
| 28 | if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return | ||
| 29 | |||
| 30 | const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL | ||
| 31 | |||
| 32 | logger.info('Auto follow instances of index %s.', indexUrl) | ||
| 33 | |||
| 34 | try { | ||
| 35 | const serverActor = await getServerActor() | ||
| 36 | |||
| 37 | const searchParams = { count: 1000 } | ||
| 38 | if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() }) | ||
| 39 | |||
| 40 | this.lastCheck = new Date() | ||
| 41 | |||
| 42 | const { body } = await doJSONRequest<any>(indexUrl, { searchParams }) | ||
| 43 | if (!body.data || Array.isArray(body.data) === false) { | ||
| 44 | logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body }) | ||
| 45 | return | ||
| 46 | } | ||
| 47 | |||
| 48 | const hosts: string[] = body.data.map(o => o.host) | ||
| 49 | const chunks = chunk(hosts, 20) | ||
| 50 | |||
| 51 | for (const chunk of chunks) { | ||
| 52 | const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk) | ||
| 53 | |||
| 54 | for (const unfollowedHost of unfollowedHosts) { | ||
| 55 | const payload = { | ||
| 56 | host: unfollowedHost, | ||
| 57 | name: SERVER_ACTOR_NAME, | ||
| 58 | followerActorId: serverActor.id, | ||
| 59 | isAutoFollow: true | ||
| 60 | } | ||
| 61 | |||
| 62 | JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) | ||
| 63 | } | ||
| 64 | } | ||
| 65 | |||
| 66 | } catch (err) { | ||
| 67 | logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err }) | ||
| 68 | } | ||
| 69 | |||
| 70 | } | ||
| 71 | |||
| 72 | static get Instance () { | ||
| 73 | return this.instance || (this.instance = new this()) | ||
| 74 | } | ||
| 75 | } | ||
diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts deleted file mode 100644 index b06f5a9b5..000000000 --- a/server/lib/schedulers/geo-ip-update-scheduler.ts +++ /dev/null | |||
| @@ -1,22 +0,0 @@ | |||
| 1 | import { GeoIP } from '@server/helpers/geo-ip' | ||
| 2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 3 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 4 | |||
| 5 | export class GeoIPUpdateScheduler extends AbstractScheduler { | ||
| 6 | |||
| 7 | private static instance: AbstractScheduler | ||
| 8 | |||
| 9 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE | ||
| 10 | |||
| 11 | private constructor () { | ||
| 12 | super() | ||
| 13 | } | ||
| 14 | |||
| 15 | protected internalExecute () { | ||
| 16 | return GeoIP.Instance.updateDatabase() | ||
| 17 | } | ||
| 18 | |||
| 19 | static get Instance () { | ||
| 20 | return this.instance || (this.instance = new this()) | ||
| 21 | } | ||
| 22 | } | ||
diff --git a/server/lib/schedulers/peertube-version-check-scheduler.ts b/server/lib/schedulers/peertube-version-check-scheduler.ts deleted file mode 100644 index bc38ed49f..000000000 --- a/server/lib/schedulers/peertube-version-check-scheduler.ts +++ /dev/null | |||
| @@ -1,55 +0,0 @@ | |||
| 1 | |||
| 2 | import { doJSONRequest } from '@server/helpers/requests' | ||
| 3 | import { ApplicationModel } from '@server/models/application/application' | ||
| 4 | import { compareSemVer } from '@shared/core-utils' | ||
| 5 | import { JoinPeerTubeVersions } from '@shared/models' | ||
| 6 | import { logger } from '../../helpers/logger' | ||
| 7 | import { CONFIG } from '../../initializers/config' | ||
| 8 | import { PEERTUBE_VERSION, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 9 | import { Notifier } from '../notifier' | ||
| 10 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 11 | |||
| 12 | export class PeerTubeVersionCheckScheduler extends AbstractScheduler { | ||
| 13 | |||
| 14 | private static instance: AbstractScheduler | ||
| 15 | |||
| 16 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION | ||
| 17 | |||
| 18 | private constructor () { | ||
| 19 | super() | ||
| 20 | } | ||
| 21 | |||
| 22 | protected async internalExecute () { | ||
| 23 | return this.checkLatestVersion() | ||
| 24 | } | ||
| 25 | |||
| 26 | private async checkLatestVersion () { | ||
| 27 | if (CONFIG.PEERTUBE.CHECK_LATEST_VERSION.ENABLED === false) return | ||
| 28 | |||
| 29 | logger.info('Checking latest PeerTube version.') | ||
| 30 | |||
| 31 | const { body } = await doJSONRequest<JoinPeerTubeVersions>(CONFIG.PEERTUBE.CHECK_LATEST_VERSION.URL) | ||
| 32 | |||
| 33 | if (!body?.peertube?.latestVersion) { | ||
| 34 | logger.warn('Cannot check latest PeerTube version: body is invalid.', { body }) | ||
| 35 | return | ||
| 36 | } | ||
| 37 | |||
| 38 | const latestVersion = body.peertube.latestVersion | ||
| 39 | const application = await ApplicationModel.load() | ||
| 40 | |||
| 41 | // Already checked this version | ||
| 42 | if (application.latestPeerTubeVersion === latestVersion) return | ||
| 43 | |||
| 44 | if (compareSemVer(PEERTUBE_VERSION, latestVersion) < 0) { | ||
| 45 | application.latestPeerTubeVersion = latestVersion | ||
| 46 | await application.save() | ||
| 47 | |||
| 48 | Notifier.Instance.notifyOfNewPeerTubeVersion(application, latestVersion) | ||
| 49 | } | ||
| 50 | } | ||
| 51 | |||
| 52 | static get Instance () { | ||
| 53 | return this.instance || (this.instance = new this()) | ||
| 54 | } | ||
| 55 | } | ||
diff --git a/server/lib/schedulers/plugins-check-scheduler.ts b/server/lib/schedulers/plugins-check-scheduler.ts deleted file mode 100644 index 820c01693..000000000 --- a/server/lib/schedulers/plugins-check-scheduler.ts +++ /dev/null | |||
| @@ -1,74 +0,0 @@ | |||
| 1 | import { chunk } from 'lodash' | ||
| 2 | import { compareSemVer } from '@shared/core-utils' | ||
| 3 | import { logger } from '../../helpers/logger' | ||
| 4 | import { CONFIG } from '../../initializers/config' | ||
| 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 6 | import { PluginModel } from '../../models/server/plugin' | ||
| 7 | import { Notifier } from '../notifier' | ||
| 8 | import { getLatestPluginsVersion } from '../plugins/plugin-index' | ||
| 9 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 10 | |||
| 11 | export class PluginsCheckScheduler extends AbstractScheduler { | ||
| 12 | |||
| 13 | private static instance: AbstractScheduler | ||
| 14 | |||
| 15 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PLUGINS | ||
| 16 | |||
| 17 | private constructor () { | ||
| 18 | super() | ||
| 19 | } | ||
| 20 | |||
| 21 | protected async internalExecute () { | ||
| 22 | return this.checkLatestPluginsVersion() | ||
| 23 | } | ||
| 24 | |||
| 25 | private async checkLatestPluginsVersion () { | ||
| 26 | if (CONFIG.PLUGINS.INDEX.ENABLED === false) return | ||
| 27 | |||
| 28 | logger.info('Checking latest plugins version.') | ||
| 29 | |||
| 30 | const plugins = await PluginModel.listInstalled() | ||
| 31 | |||
| 32 | // Process 10 plugins in 1 HTTP request | ||
| 33 | const chunks = chunk(plugins, 10) | ||
| 34 | for (const chunk of chunks) { | ||
| 35 | // Find plugins according to their npm name | ||
| 36 | const pluginIndex: { [npmName: string]: PluginModel } = {} | ||
| 37 | for (const plugin of chunk) { | ||
| 38 | pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin | ||
| 39 | } | ||
| 40 | |||
| 41 | const npmNames = Object.keys(pluginIndex) | ||
| 42 | |||
| 43 | try { | ||
| 44 | const results = await getLatestPluginsVersion(npmNames) | ||
| 45 | |||
| 46 | for (const result of results) { | ||
| 47 | const plugin = pluginIndex[result.npmName] | ||
| 48 | if (!result.latestVersion) continue | ||
| 49 | |||
| 50 | if ( | ||
| 51 | !plugin.latestVersion || | ||
| 52 | (plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0) | ||
| 53 | ) { | ||
| 54 | plugin.latestVersion = result.latestVersion | ||
| 55 | await plugin.save() | ||
| 56 | |||
| 57 | // Notify if there is an higher plugin version available | ||
| 58 | if (compareSemVer(plugin.version, result.latestVersion) < 0) { | ||
| 59 | Notifier.Instance.notifyOfNewPluginVersion(plugin) | ||
| 60 | } | ||
| 61 | |||
| 62 | logger.info('Plugin %s has a new latest version %s.', result.npmName, plugin.latestVersion) | ||
| 63 | } | ||
| 64 | } | ||
| 65 | } catch (err) { | ||
| 66 | logger.error('Cannot get latest plugins version.', { npmNames, err }) | ||
| 67 | } | ||
| 68 | } | ||
| 69 | } | ||
| 70 | |||
| 71 | static get Instance () { | ||
| 72 | return this.instance || (this.instance = new this()) | ||
| 73 | } | ||
| 74 | } | ||
diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts deleted file mode 100644 index 61e93eafa..000000000 --- a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts +++ /dev/null | |||
| @@ -1,40 +0,0 @@ | |||
| 1 | |||
| 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 3 | import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' | ||
| 4 | import { uploadx } from '../uploadx' | ||
| 5 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 6 | |||
| 7 | const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') | ||
| 8 | |||
| 9 | export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { | ||
| 10 | |||
| 11 | private static instance: AbstractScheduler | ||
| 12 | private lastExecutionTimeMs: number | ||
| 13 | |||
| 14 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS | ||
| 15 | |||
| 16 | private constructor () { | ||
| 17 | super() | ||
| 18 | |||
| 19 | this.lastExecutionTimeMs = new Date().getTime() | ||
| 20 | } | ||
| 21 | |||
| 22 | protected async internalExecute () { | ||
| 23 | logger.debug('Removing dangling resumable uploads', lTags()) | ||
| 24 | |||
| 25 | const now = new Date().getTime() | ||
| 26 | |||
| 27 | try { | ||
| 28 | // Remove files that were not updated since the last execution | ||
| 29 | await uploadx.storage.purge(now - this.lastExecutionTimeMs) | ||
| 30 | } catch (error) { | ||
| 31 | logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) | ||
| 32 | } finally { | ||
| 33 | this.lastExecutionTimeMs = now | ||
| 34 | } | ||
| 35 | } | ||
| 36 | |||
| 37 | static get Instance () { | ||
| 38 | return this.instance || (this.instance = new this()) | ||
| 39 | } | ||
| 40 | } | ||
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts deleted file mode 100644 index 34b160799..000000000 --- a/server/lib/schedulers/remove-old-history-scheduler.ts +++ /dev/null | |||
| @@ -1,31 +0,0 @@ | |||
| 1 | import { logger } from '../../helpers/logger' | ||
| 2 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 3 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 4 | import { UserVideoHistoryModel } from '../../models/user/user-video-history' | ||
| 5 | import { CONFIG } from '../../initializers/config' | ||
| 6 | |||
| 7 | export class RemoveOldHistoryScheduler extends AbstractScheduler { | ||
| 8 | |||
| 9 | private static instance: AbstractScheduler | ||
| 10 | |||
| 11 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY | ||
| 12 | |||
| 13 | private constructor () { | ||
| 14 | super() | ||
| 15 | } | ||
| 16 | |||
| 17 | protected internalExecute () { | ||
| 18 | if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return | ||
| 19 | |||
| 20 | logger.info('Removing old videos history.') | ||
| 21 | |||
| 22 | const now = new Date() | ||
| 23 | const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString() | ||
| 24 | |||
| 25 | return UserVideoHistoryModel.removeOldHistory(beforeDate) | ||
| 26 | } | ||
| 27 | |||
| 28 | static get Instance () { | ||
| 29 | return this.instance || (this.instance = new this()) | ||
| 30 | } | ||
| 31 | } | ||
diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts deleted file mode 100644 index 8bc53a045..000000000 --- a/server/lib/schedulers/remove-old-views-scheduler.ts +++ /dev/null | |||
| @@ -1,31 +0,0 @@ | |||
| 1 | import { VideoViewModel } from '@server/models/view/video-view' | ||
| 2 | import { logger } from '../../helpers/logger' | ||
| 3 | import { CONFIG } from '../../initializers/config' | ||
| 4 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 5 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 6 | |||
| 7 | export class RemoveOldViewsScheduler extends AbstractScheduler { | ||
| 8 | |||
| 9 | private static instance: AbstractScheduler | ||
| 10 | |||
| 11 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS | ||
| 12 | |||
| 13 | private constructor () { | ||
| 14 | super() | ||
| 15 | } | ||
| 16 | |||
| 17 | protected internalExecute () { | ||
| 18 | if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return | ||
| 19 | |||
| 20 | logger.info('Removing old videos views.') | ||
| 21 | |||
| 22 | const now = new Date() | ||
| 23 | const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString() | ||
| 24 | |||
| 25 | return VideoViewModel.removeOldRemoteViewsHistory(beforeDate) | ||
| 26 | } | ||
| 27 | |||
| 28 | static get Instance () { | ||
| 29 | return this.instance || (this.instance = new this()) | ||
| 30 | } | ||
| 31 | } | ||
diff --git a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts deleted file mode 100644 index f7a26d2bc..000000000 --- a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts +++ /dev/null | |||
| @@ -1,42 +0,0 @@ | |||
| 1 | import { CONFIG } from '@server/initializers/config' | ||
| 2 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
| 3 | import { logger, loggerTagsFactory } from '../../helpers/logger' | ||
| 4 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 5 | import { getRunnerJobHandlerClass } from '../runners' | ||
| 6 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 7 | |||
| 8 | const lTags = loggerTagsFactory('runner') | ||
| 9 | |||
| 10 | export class RunnerJobWatchDogScheduler extends AbstractScheduler { | ||
| 11 | |||
| 12 | private static instance: AbstractScheduler | ||
| 13 | |||
| 14 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG | ||
| 15 | |||
| 16 | private constructor () { | ||
| 17 | super() | ||
| 18 | } | ||
| 19 | |||
| 20 | protected async internalExecute () { | ||
| 21 | const vodStalledJobs = await RunnerJobModel.listStalledJobs({ | ||
| 22 | staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, | ||
| 23 | types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ] | ||
| 24 | }) | ||
| 25 | |||
| 26 | const liveStalledJobs = await RunnerJobModel.listStalledJobs({ | ||
| 27 | staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE, | ||
| 28 | types: [ 'live-rtmp-hls-transcoding' ] | ||
| 29 | }) | ||
| 30 | |||
| 31 | for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) { | ||
| 32 | logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type)) | ||
| 33 | |||
| 34 | const Handler = getRunnerJobHandlerClass(stalled) | ||
| 35 | await new Handler().abort({ runnerJob: stalled }) | ||
| 36 | } | ||
| 37 | } | ||
| 38 | |||
| 39 | static get Instance () { | ||
| 40 | return this.instance || (this.instance = new this()) | ||
| 41 | } | ||
| 42 | } | ||
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts deleted file mode 100644 index e38685c04..000000000 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ /dev/null | |||
| @@ -1,89 +0,0 @@ | |||
| 1 | import { VideoModel } from '@server/models/video/video' | ||
| 2 | import { MScheduleVideoUpdate } from '@server/types/models' | ||
| 3 | import { VideoPrivacy, VideoState } from '@shared/models' | ||
| 4 | import { logger } from '../../helpers/logger' | ||
| 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 6 | import { sequelizeTypescript } from '../../initializers/database' | ||
| 7 | import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' | ||
| 8 | import { Notifier } from '../notifier' | ||
| 9 | import { addVideoJobsAfterUpdate } from '../video' | ||
| 10 | import { VideoPathManager } from '../video-path-manager' | ||
| 11 | import { setVideoPrivacy } from '../video-privacy' | ||
| 12 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 13 | |||
| 14 | export class UpdateVideosScheduler extends AbstractScheduler { | ||
| 15 | |||
| 16 | private static instance: AbstractScheduler | ||
| 17 | |||
| 18 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS | ||
| 19 | |||
| 20 | private constructor () { | ||
| 21 | super() | ||
| 22 | } | ||
| 23 | |||
| 24 | protected async internalExecute () { | ||
| 25 | return this.updateVideos() | ||
| 26 | } | ||
| 27 | |||
| 28 | private async updateVideos () { | ||
| 29 | if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined | ||
| 30 | |||
| 31 | const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() | ||
| 32 | |||
| 33 | for (const schedule of schedules) { | ||
| 34 | const videoOnly = await VideoModel.load(schedule.videoId) | ||
| 35 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid) | ||
| 36 | |||
| 37 | try { | ||
| 38 | const { video, published } = await this.updateAVideo(schedule) | ||
| 39 | |||
| 40 | if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video) | ||
| 41 | } catch (err) { | ||
| 42 | logger.error('Cannot update video', { err }) | ||
| 43 | } | ||
| 44 | |||
| 45 | mutexReleaser() | ||
| 46 | } | ||
| 47 | } | ||
| 48 | |||
| 49 | private async updateAVideo (schedule: MScheduleVideoUpdate) { | ||
| 50 | let oldPrivacy: VideoPrivacy | ||
| 51 | let isNewVideo: boolean | ||
| 52 | let published = false | ||
| 53 | |||
| 54 | const video = await sequelizeTypescript.transaction(async t => { | ||
| 55 | const video = await VideoModel.loadFull(schedule.videoId, t) | ||
| 56 | if (video.state === VideoState.TO_TRANSCODE) return null | ||
| 57 | |||
| 58 | logger.info('Executing scheduled video update on %s.', video.uuid) | ||
| 59 | |||
| 60 | if (schedule.privacy) { | ||
| 61 | isNewVideo = video.isNewVideo(schedule.privacy) | ||
| 62 | oldPrivacy = video.privacy | ||
| 63 | |||
| 64 | setVideoPrivacy(video, schedule.privacy) | ||
| 65 | await video.save({ transaction: t }) | ||
| 66 | |||
| 67 | if (oldPrivacy === VideoPrivacy.PRIVATE) { | ||
| 68 | published = true | ||
| 69 | } | ||
| 70 | } | ||
| 71 | |||
| 72 | await schedule.destroy({ transaction: t }) | ||
| 73 | |||
| 74 | return video | ||
| 75 | }) | ||
| 76 | |||
| 77 | if (!video) { | ||
| 78 | return { video, published: false } | ||
| 79 | } | ||
| 80 | |||
| 81 | await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false }) | ||
| 82 | |||
| 83 | return { video, published } | ||
| 84 | } | ||
| 85 | |||
| 86 | static get Instance () { | ||
| 87 | return this.instance || (this.instance = new this()) | ||
| 88 | } | ||
| 89 | } | ||
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts deleted file mode 100644 index efb957fac..000000000 --- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ /dev/null | |||
| @@ -1,50 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
| 4 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | ||
| 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 6 | import { synchronizeChannel } from '../sync-channel' | ||
| 7 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 8 | |||
| 9 | export class VideoChannelSyncLatestScheduler extends AbstractScheduler { | ||
| 10 | private static instance: AbstractScheduler | ||
| 11 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL | ||
| 12 | |||
| 13 | private constructor () { | ||
| 14 | super() | ||
| 15 | } | ||
| 16 | |||
| 17 | protected async internalExecute () { | ||
| 18 | if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { | ||
| 19 | logger.debug('Discard channels synchronization as the feature is disabled') | ||
| 20 | return | ||
| 21 | } | ||
| 22 | |||
| 23 | logger.info('Checking channels to synchronize') | ||
| 24 | |||
| 25 | const channelSyncs = await VideoChannelSyncModel.listSyncs() | ||
| 26 | |||
| 27 | for (const sync of channelSyncs) { | ||
| 28 | const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) | ||
| 29 | |||
| 30 | logger.info( | ||
| 31 | 'Creating video import jobs for "%s" sync with external channel "%s"', | ||
| 32 | channel.Actor.preferredUsername, sync.externalChannelUrl | ||
| 33 | ) | ||
| 34 | |||
| 35 | const onlyAfter = sync.lastSyncAt || sync.createdAt | ||
| 36 | |||
| 37 | await synchronizeChannel({ | ||
| 38 | channel, | ||
| 39 | externalChannelUrl: sync.externalChannelUrl, | ||
| 40 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, | ||
| 41 | channelSync: sync, | ||
| 42 | onlyAfter | ||
| 43 | }) | ||
| 44 | } | ||
| 45 | } | ||
| 46 | |||
| 47 | static get Instance () { | ||
| 48 | return this.instance || (this.instance = new this()) | ||
| 49 | } | ||
| 50 | } | ||
diff --git a/server/lib/schedulers/video-views-buffer-scheduler.ts b/server/lib/schedulers/video-views-buffer-scheduler.ts deleted file mode 100644 index 244a88b14..000000000 --- a/server/lib/schedulers/video-views-buffer-scheduler.ts +++ /dev/null | |||
| @@ -1,52 +0,0 @@ | |||
| 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 2 | import { VideoModel } from '@server/models/video/video' | ||
| 3 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 4 | import { federateVideoIfNeeded } from '../activitypub/videos' | ||
| 5 | import { Redis } from '../redis' | ||
| 6 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 7 | |||
| 8 | const lTags = loggerTagsFactory('views') | ||
| 9 | |||
| 10 | export class VideoViewsBufferScheduler extends AbstractScheduler { | ||
| 11 | |||
| 12 | private static instance: AbstractScheduler | ||
| 13 | |||
| 14 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.VIDEO_VIEWS_BUFFER_UPDATE | ||
| 15 | |||
| 16 | private constructor () { | ||
| 17 | super() | ||
| 18 | } | ||
| 19 | |||
| 20 | protected async internalExecute () { | ||
| 21 | const videoIds = await Redis.Instance.listLocalVideosViewed() | ||
| 22 | if (videoIds.length === 0) return | ||
| 23 | |||
| 24 | for (const videoId of videoIds) { | ||
| 25 | try { | ||
| 26 | const views = await Redis.Instance.getLocalVideoViews(videoId) | ||
| 27 | await Redis.Instance.deleteLocalVideoViews(videoId) | ||
| 28 | |||
| 29 | const video = await VideoModel.loadFull(videoId) | ||
| 30 | if (!video) { | ||
| 31 | logger.debug('Video %d does not exist anymore, skipping videos view addition.', videoId, lTags()) | ||
| 32 | continue | ||
| 33 | } | ||
| 34 | |||
| 35 | logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid)) | ||
| 36 | |||
| 37 | // If this is a remote video, the origin instance will send us an update | ||
| 38 | await VideoModel.incrementViews(videoId, views) | ||
| 39 | |||
| 40 | // Send video update | ||
| 41 | video.views += views | ||
| 42 | await federateVideoIfNeeded(video, false) | ||
| 43 | } catch (err) { | ||
| 44 | logger.error('Cannot process local video views buffer of video %d.', videoId, { err, ...lTags() }) | ||
| 45 | } | ||
| 46 | } | ||
| 47 | } | ||
| 48 | |||
| 49 | static get Instance () { | ||
| 50 | return this.instance || (this.instance = new this()) | ||
| 51 | } | ||
| 52 | } | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts deleted file mode 100644 index 91625ccb5..000000000 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ /dev/null | |||
| @@ -1,375 +0,0 @@ | |||
| 1 | import { move } from 'fs-extra' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { getServerActor } from '@server/models/application/application' | ||
| 4 | import { VideoModel } from '@server/models/video/video' | ||
| 5 | import { | ||
| 6 | MStreamingPlaylistFiles, | ||
| 7 | MVideoAccountLight, | ||
| 8 | MVideoFile, | ||
| 9 | MVideoFileVideo, | ||
| 10 | MVideoRedundancyFileVideo, | ||
| 11 | MVideoRedundancyStreamingPlaylistVideo, | ||
| 12 | MVideoRedundancyVideo, | ||
| 13 | MVideoWithAllFiles | ||
| 14 | } from '@server/types/models' | ||
| 15 | import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' | ||
| 16 | import { logger, loggerTagsFactory } from '../../helpers/logger' | ||
| 17 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | ||
| 18 | import { CONFIG } from '../../initializers/config' | ||
| 19 | import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' | ||
| 20 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | ||
| 21 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | ||
| 22 | import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' | ||
| 23 | import { getOrCreateAPVideo } from '../activitypub/videos' | ||
| 24 | import { downloadPlaylistSegments } from '../hls' | ||
| 25 | import { removeVideoRedundancy } from '../redundancy' | ||
| 26 | import { generateHLSRedundancyUrl, generateWebVideoRedundancyUrl } from '../video-urls' | ||
| 27 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 28 | |||
| 29 | const lTags = loggerTagsFactory('redundancy') | ||
| 30 | |||
| 31 | type CandidateToDuplicate = { | ||
| 32 | redundancy: VideosRedundancyStrategy | ||
| 33 | video: MVideoWithAllFiles | ||
| 34 | files: MVideoFile[] | ||
| 35 | streamingPlaylists: MStreamingPlaylistFiles[] | ||
| 36 | } | ||
| 37 | |||
| 38 | function isMVideoRedundancyFileVideo ( | ||
| 39 | o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo | ||
| 40 | ): o is MVideoRedundancyFileVideo { | ||
| 41 | return !!(o as MVideoRedundancyFileVideo).VideoFile | ||
| 42 | } | ||
| 43 | |||
| 44 | export class VideosRedundancyScheduler extends AbstractScheduler { | ||
| 45 | |||
| 46 | private static instance: VideosRedundancyScheduler | ||
| 47 | |||
| 48 | protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL | ||
| 49 | |||
| 50 | private constructor () { | ||
| 51 | super() | ||
| 52 | } | ||
| 53 | |||
| 54 | async createManualRedundancy (videoId: number) { | ||
| 55 | const videoToDuplicate = await VideoModel.loadWithFiles(videoId) | ||
| 56 | |||
| 57 | if (!videoToDuplicate) { | ||
| 58 | logger.warn('Video to manually duplicate %d does not exist anymore.', videoId, lTags()) | ||
| 59 | return | ||
| 60 | } | ||
| 61 | |||
| 62 | return this.createVideoRedundancies({ | ||
| 63 | video: videoToDuplicate, | ||
| 64 | redundancy: null, | ||
| 65 | files: videoToDuplicate.VideoFiles, | ||
| 66 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
| 67 | }) | ||
| 68 | } | ||
| 69 | |||
| 70 | protected async internalExecute () { | ||
| 71 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | ||
| 72 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy, lTags()) | ||
| 73 | |||
| 74 | try { | ||
| 75 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) | ||
| 76 | if (!videoToDuplicate) continue | ||
| 77 | |||
| 78 | const candidateToDuplicate = { | ||
| 79 | video: videoToDuplicate, | ||
| 80 | redundancy: redundancyConfig, | ||
| 81 | files: videoToDuplicate.VideoFiles, | ||
| 82 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
| 83 | } | ||
| 84 | |||
| 85 | await this.purgeCacheIfNeeded(candidateToDuplicate) | ||
| 86 | |||
| 87 | if (await this.isTooHeavy(candidateToDuplicate)) { | ||
| 88 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url, lTags(videoToDuplicate.uuid)) | ||
| 89 | continue | ||
| 90 | } | ||
| 91 | |||
| 92 | logger.info( | ||
| 93 | 'Will duplicate video %s in redundancy scheduler "%s".', | ||
| 94 | videoToDuplicate.url, redundancyConfig.strategy, lTags(videoToDuplicate.uuid) | ||
| 95 | ) | ||
| 96 | |||
| 97 | await this.createVideoRedundancies(candidateToDuplicate) | ||
| 98 | } catch (err) { | ||
| 99 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err, ...lTags() }) | ||
| 100 | } | ||
| 101 | } | ||
| 102 | |||
| 103 | await this.extendsLocalExpiration() | ||
| 104 | |||
| 105 | await this.purgeRemoteExpired() | ||
| 106 | } | ||
| 107 | |||
| 108 | static get Instance () { | ||
| 109 | return this.instance || (this.instance = new this()) | ||
| 110 | } | ||
| 111 | |||
| 112 | private async extendsLocalExpiration () { | ||
| 113 | const expired = await VideoRedundancyModel.listLocalExpired() | ||
| 114 | |||
| 115 | for (const redundancyModel of expired) { | ||
| 116 | try { | ||
| 117 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | ||
| 118 | |||
| 119 | // If the admin disabled the redundancy, remove this redundancy instead of extending it | ||
| 120 | if (!redundancyConfig) { | ||
| 121 | logger.info( | ||
| 122 | 'Destroying redundancy %s because the redundancy %s does not exist anymore.', | ||
| 123 | redundancyModel.url, redundancyModel.strategy | ||
| 124 | ) | ||
| 125 | |||
| 126 | await removeVideoRedundancy(redundancyModel) | ||
| 127 | continue | ||
| 128 | } | ||
| 129 | |||
| 130 | const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy) | ||
| 131 | |||
| 132 | // If the admin decreased the cache size, remove this redundancy instead of extending it | ||
| 133 | if (totalUsed > redundancyConfig.size) { | ||
| 134 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | ||
| 135 | |||
| 136 | await removeVideoRedundancy(redundancyModel) | ||
| 137 | continue | ||
| 138 | } | ||
| 139 | |||
| 140 | await this.extendsRedundancy(redundancyModel) | ||
| 141 | } catch (err) { | ||
| 142 | logger.error( | ||
| 143 | 'Cannot extend or remove expiration of %s video from our redundancy system.', | ||
| 144 | this.buildEntryLogId(redundancyModel), { err, ...lTags(redundancyModel.getVideoUUID()) } | ||
| 145 | ) | ||
| 146 | } | ||
| 147 | } | ||
| 148 | } | ||
| 149 | |||
| 150 | private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) { | ||
| 151 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | ||
| 152 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration | ||
| 153 | if (!redundancy) { | ||
| 154 | await removeVideoRedundancy(redundancyModel) | ||
| 155 | return | ||
| 156 | } | ||
| 157 | |||
| 158 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) | ||
| 159 | } | ||
| 160 | |||
| 161 | private async purgeRemoteExpired () { | ||
| 162 | const expired = await VideoRedundancyModel.listRemoteExpired() | ||
| 163 | |||
| 164 | for (const redundancyModel of expired) { | ||
| 165 | try { | ||
| 166 | await removeVideoRedundancy(redundancyModel) | ||
| 167 | } catch (err) { | ||
| 168 | logger.error( | ||
| 169 | 'Cannot remove redundancy %s from our redundancy system.', | ||
| 170 | this.buildEntryLogId(redundancyModel), lTags(redundancyModel.getVideoUUID()) | ||
| 171 | ) | ||
| 172 | } | ||
| 173 | } | ||
| 174 | } | ||
| 175 | |||
| 176 | private findVideoToDuplicate (cache: VideosRedundancyStrategy) { | ||
| 177 | if (cache.strategy === 'most-views') { | ||
| 178 | return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | ||
| 179 | } | ||
| 180 | |||
| 181 | if (cache.strategy === 'trending') { | ||
| 182 | return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | ||
| 183 | } | ||
| 184 | |||
| 185 | if (cache.strategy === 'recently-added') { | ||
| 186 | const minViews = cache.minViews | ||
| 187 | return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) | ||
| 188 | } | ||
| 189 | } | ||
| 190 | |||
| 191 | private async createVideoRedundancies (data: CandidateToDuplicate) { | ||
| 192 | const video = await this.loadAndRefreshVideo(data.video.url) | ||
| 193 | |||
| 194 | if (!video) { | ||
| 195 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url, lTags(data.video.uuid)) | ||
| 196 | |||
| 197 | return | ||
| 198 | } | ||
| 199 | |||
| 200 | for (const file of data.files) { | ||
| 201 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) | ||
| 202 | if (existingRedundancy) { | ||
| 203 | await this.extendsRedundancy(existingRedundancy) | ||
| 204 | |||
| 205 | continue | ||
| 206 | } | ||
| 207 | |||
| 208 | await this.createVideoFileRedundancy(data.redundancy, video, file) | ||
| 209 | } | ||
| 210 | |||
| 211 | for (const streamingPlaylist of data.streamingPlaylists) { | ||
| 212 | const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) | ||
| 213 | if (existingRedundancy) { | ||
| 214 | await this.extendsRedundancy(existingRedundancy) | ||
| 215 | |||
| 216 | continue | ||
| 217 | } | ||
| 218 | |||
| 219 | await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) | ||
| 220 | } | ||
| 221 | } | ||
| 222 | |||
| 223 | private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { | ||
| 224 | let strategy = 'manual' | ||
| 225 | let expiresOn: Date = null | ||
| 226 | |||
| 227 | if (redundancy) { | ||
| 228 | strategy = redundancy.strategy | ||
| 229 | expiresOn = this.buildNewExpiration(redundancy.minLifetime) | ||
| 230 | } | ||
| 231 | |||
| 232 | const file = fileArg as MVideoFileVideo | ||
| 233 | file.Video = video | ||
| 234 | |||
| 235 | const serverActor = await getServerActor() | ||
| 236 | |||
| 237 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) | ||
| 238 | |||
| 239 | const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT) | ||
| 240 | |||
| 241 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) | ||
| 242 | await move(tmpPath, destPath, { overwrite: true }) | ||
| 243 | |||
| 244 | const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ | ||
| 245 | expiresOn, | ||
| 246 | url: getLocalVideoCacheFileActivityPubUrl(file), | ||
| 247 | fileUrl: generateWebVideoRedundancyUrl(file), | ||
| 248 | strategy, | ||
| 249 | videoFileId: file.id, | ||
| 250 | actorId: serverActor.id | ||
| 251 | }) | ||
| 252 | |||
| 253 | createdModel.VideoFile = file | ||
| 254 | |||
| 255 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
| 256 | |||
| 257 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url, lTags(video.uuid)) | ||
| 258 | } | ||
| 259 | |||
| 260 | private async createStreamingPlaylistRedundancy ( | ||
| 261 | redundancy: VideosRedundancyStrategy, | ||
| 262 | video: MVideoAccountLight, | ||
| 263 | playlistArg: MStreamingPlaylistFiles | ||
| 264 | ) { | ||
| 265 | let strategy = 'manual' | ||
| 266 | let expiresOn: Date = null | ||
| 267 | |||
| 268 | if (redundancy) { | ||
| 269 | strategy = redundancy.strategy | ||
| 270 | expiresOn = this.buildNewExpiration(redundancy.minLifetime) | ||
| 271 | } | ||
| 272 | |||
| 273 | const playlist = Object.assign(playlistArg, { Video: video }) | ||
| 274 | const serverActor = await getServerActor() | ||
| 275 | |||
| 276 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) | ||
| 277 | |||
| 278 | const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) | ||
| 279 | const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) | ||
| 280 | |||
| 281 | const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 | ||
| 282 | const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance | ||
| 283 | await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB) | ||
| 284 | |||
| 285 | const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ | ||
| 286 | expiresOn, | ||
| 287 | url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | ||
| 288 | fileUrl: generateHLSRedundancyUrl(video, playlistArg), | ||
| 289 | strategy, | ||
| 290 | videoStreamingPlaylistId: playlist.id, | ||
| 291 | actorId: serverActor.id | ||
| 292 | }) | ||
| 293 | |||
| 294 | createdModel.VideoStreamingPlaylist = playlist | ||
| 295 | |||
| 296 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
| 297 | |||
| 298 | logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url, lTags(video.uuid)) | ||
| 299 | } | ||
| 300 | |||
| 301 | private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { | ||
| 302 | logger.info('Extending expiration of %s.', redundancy.url, lTags(redundancy.getVideoUUID())) | ||
| 303 | |||
| 304 | const serverActor = await getServerActor() | ||
| 305 | |||
| 306 | redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs) | ||
| 307 | await redundancy.save() | ||
| 308 | |||
| 309 | await sendUpdateCacheFile(serverActor, redundancy) | ||
| 310 | } | ||
| 311 | |||
| 312 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { | ||
| 313 | while (await this.isTooHeavy(candidateToDuplicate)) { | ||
| 314 | const redundancy = candidateToDuplicate.redundancy | ||
| 315 | const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) | ||
| 316 | if (!toDelete) return | ||
| 317 | |||
| 318 | const videoId = toDelete.VideoFile | ||
| 319 | ? toDelete.VideoFile.videoId | ||
| 320 | : toDelete.VideoStreamingPlaylist.videoId | ||
| 321 | |||
| 322 | const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId) | ||
| 323 | |||
| 324 | for (const redundancy of redundancies) { | ||
| 325 | await removeVideoRedundancy(redundancy) | ||
| 326 | } | ||
| 327 | } | ||
| 328 | } | ||
| 329 | |||
| 330 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { | ||
| 331 | const maxSize = candidateToDuplicate.redundancy.size | ||
| 332 | |||
| 333 | const { totalUsed: alreadyUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) | ||
| 334 | |||
| 335 | const videoSize = this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) | ||
| 336 | const willUse = alreadyUsed + videoSize | ||
| 337 | |||
| 338 | logger.debug('Checking candidate size.', { maxSize, alreadyUsed, videoSize, willUse, ...lTags(candidateToDuplicate.video.uuid) }) | ||
| 339 | |||
| 340 | return willUse > maxSize | ||
| 341 | } | ||
| 342 | |||
| 343 | private buildNewExpiration (expiresAfterMs: number) { | ||
| 344 | return new Date(Date.now() + expiresAfterMs) | ||
| 345 | } | ||
| 346 | |||
| 347 | private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { | ||
| 348 | if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | ||
| 349 | |||
| 350 | return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` | ||
| 351 | } | ||
| 352 | |||
| 353 | private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number { | ||
| 354 | const fileReducer = (previous: number, current: MVideoFile) => previous + current.size | ||
| 355 | |||
| 356 | let allFiles = files | ||
| 357 | for (const p of playlists) { | ||
| 358 | allFiles = allFiles.concat(p.VideoFiles) | ||
| 359 | } | ||
| 360 | |||
| 361 | return allFiles.reduce(fileReducer, 0) | ||
| 362 | } | ||
| 363 | |||
| 364 | private async loadAndRefreshVideo (videoUrl: string) { | ||
| 365 | // We need more attributes and check if the video still exists | ||
| 366 | const getVideoOptions = { | ||
| 367 | videoObject: videoUrl, | ||
| 368 | syncParam: { rates: false, shares: false, comments: false, refreshVideo: true }, | ||
| 369 | fetchType: 'all' as 'all' | ||
| 370 | } | ||
| 371 | const { video } = await getOrCreateAPVideo(getVideoOptions) | ||
| 372 | |||
| 373 | return video | ||
| 374 | } | ||
| 375 | } | ||
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts deleted file mode 100644 index 1ee4ae1b2..000000000 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ /dev/null | |||
| @@ -1,22 +0,0 @@ | |||
| 1 | import { YoutubeDLCLI } from '@server/helpers/youtube-dl' | ||
| 2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
| 3 | import { AbstractScheduler } from './abstract-scheduler' | ||
| 4 | |||
| 5 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { | ||
| 6 | |||
| 7 | private static instance: AbstractScheduler | ||
| 8 | |||
| 9 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE | ||
| 10 | |||
| 11 | private constructor () { | ||
| 12 | super() | ||
| 13 | } | ||
| 14 | |||
| 15 | protected internalExecute () { | ||
| 16 | return YoutubeDLCLI.updateYoutubeDLBinary() | ||
| 17 | } | ||
| 18 | |||
| 19 | static get Instance () { | ||
| 20 | return this.instance || (this.instance = new this()) | ||
| 21 | } | ||
| 22 | } | ||
diff --git a/server/lib/search.ts b/server/lib/search.ts deleted file mode 100644 index b3363fbec..000000000 --- a/server/lib/search.ts +++ /dev/null | |||
| @@ -1,49 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { AccountBlocklistModel } from '@server/models/account/account-blocklist' | ||
| 4 | import { getServerActor } from '@server/models/application/application' | ||
| 5 | import { ServerBlocklistModel } from '@server/models/server/server-blocklist' | ||
| 6 | import { SearchTargetQuery } from '@shared/models' | ||
| 7 | |||
| 8 | function isSearchIndexSearch (query: SearchTargetQuery) { | ||
| 9 | if (query.searchTarget === 'search-index') return true | ||
| 10 | |||
| 11 | const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX | ||
| 12 | |||
| 13 | if (searchIndexConfig.ENABLED !== true) return false | ||
| 14 | |||
| 15 | if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true | ||
| 16 | |||
| 17 | return false | ||
| 18 | } | ||
| 19 | |||
| 20 | async function buildMutedForSearchIndex (res: express.Response) { | ||
| 21 | const serverActor = await getServerActor() | ||
| 22 | const accountIds = [ serverActor.Account.id ] | ||
| 23 | |||
| 24 | if (res.locals.oauth) { | ||
| 25 | accountIds.push(res.locals.oauth.token.User.Account.id) | ||
| 26 | } | ||
| 27 | |||
| 28 | const [ blockedHosts, blockedAccounts ] = await Promise.all([ | ||
| 29 | ServerBlocklistModel.listHostsBlockedBy(accountIds), | ||
| 30 | AccountBlocklistModel.listHandlesBlockedBy(accountIds) | ||
| 31 | ]) | ||
| 32 | |||
| 33 | return { | ||
| 34 | blockedHosts, | ||
| 35 | blockedAccounts | ||
| 36 | } | ||
| 37 | } | ||
| 38 | |||
| 39 | function isURISearch (search: string) { | ||
| 40 | if (!search) return false | ||
| 41 | |||
| 42 | return search.startsWith('http://') || search.startsWith('https://') | ||
| 43 | } | ||
| 44 | |||
| 45 | export { | ||
| 46 | isSearchIndexSearch, | ||
| 47 | buildMutedForSearchIndex, | ||
| 48 | isURISearch | ||
| 49 | } | ||
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts deleted file mode 100644 index beb5d4d82..000000000 --- a/server/lib/server-config-manager.ts +++ /dev/null | |||
| @@ -1,384 +0,0 @@ | |||
| 1 | import { getServerCommit } from '@server/helpers/version' | ||
| 2 | import { CONFIG, isEmailEnabled } from '@server/initializers/config' | ||
| 3 | import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants' | ||
| 4 | import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup' | ||
| 5 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | ||
| 6 | import { PluginModel } from '@server/models/server/plugin' | ||
| 7 | import { HTMLServerConfig, RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models' | ||
| 8 | import { Hooks } from './plugins/hooks' | ||
| 9 | import { PluginManager } from './plugins/plugin-manager' | ||
| 10 | import { getThemeOrDefault } from './plugins/theme-utils' | ||
| 11 | import { VideoTranscodingProfilesManager } from './transcoding/default-transcoding-profiles' | ||
| 12 | |||
| 13 | /** | ||
| 14 | * | ||
| 15 | * Used to send the server config to clients (using REST/API or plugins API) | ||
| 16 | * We need a singleton class to manage config state depending on external events (to build menu entries etc) | ||
| 17 | * | ||
| 18 | */ | ||
| 19 | |||
| 20 | class ServerConfigManager { | ||
| 21 | |||
| 22 | private static instance: ServerConfigManager | ||
| 23 | |||
| 24 | private serverCommit: string | ||
| 25 | |||
| 26 | private homepageEnabled = false | ||
| 27 | |||
| 28 | private constructor () {} | ||
| 29 | |||
| 30 | async init () { | ||
| 31 | const instanceHomepage = await ActorCustomPageModel.loadInstanceHomepage() | ||
| 32 | |||
| 33 | this.updateHomepageState(instanceHomepage?.content) | ||
| 34 | } | ||
| 35 | |||
| 36 | updateHomepageState (content: string) { | ||
| 37 | this.homepageEnabled = !!content | ||
| 38 | } | ||
| 39 | |||
| 40 | async getHTMLServerConfig (): Promise<HTMLServerConfig> { | ||
| 41 | if (this.serverCommit === undefined) this.serverCommit = await getServerCommit() | ||
| 42 | |||
| 43 | const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) | ||
| 44 | |||
| 45 | return { | ||
| 46 | client: { | ||
| 47 | videos: { | ||
| 48 | miniature: { | ||
| 49 | displayAuthorAvatar: CONFIG.CLIENT.VIDEOS.MINIATURE.DISPLAY_AUTHOR_AVATAR, | ||
| 50 | preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME | ||
| 51 | }, | ||
| 52 | resumableUpload: { | ||
| 53 | maxChunkSize: CONFIG.CLIENT.VIDEOS.RESUMABLE_UPLOAD.MAX_CHUNK_SIZE | ||
| 54 | } | ||
| 55 | }, | ||
| 56 | menu: { | ||
| 57 | login: { | ||
| 58 | redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH | ||
| 59 | } | ||
| 60 | } | ||
| 61 | }, | ||
| 62 | |||
| 63 | defaults: { | ||
| 64 | publish: { | ||
| 65 | downloadEnabled: CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, | ||
| 66 | commentsEnabled: CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, | ||
| 67 | privacy: CONFIG.DEFAULTS.PUBLISH.PRIVACY, | ||
| 68 | licence: CONFIG.DEFAULTS.PUBLISH.LICENCE | ||
| 69 | }, | ||
| 70 | p2p: { | ||
| 71 | webapp: { | ||
| 72 | enabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED | ||
| 73 | }, | ||
| 74 | embed: { | ||
| 75 | enabled: CONFIG.DEFAULTS.P2P.EMBED.ENABLED | ||
| 76 | } | ||
| 77 | } | ||
| 78 | }, | ||
| 79 | |||
| 80 | webadmin: { | ||
| 81 | configuration: { | ||
| 82 | edition: { | ||
| 83 | allowed: CONFIG.WEBADMIN.CONFIGURATION.EDITION.ALLOWED | ||
| 84 | } | ||
| 85 | } | ||
| 86 | }, | ||
| 87 | |||
| 88 | instance: { | ||
| 89 | name: CONFIG.INSTANCE.NAME, | ||
| 90 | shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, | ||
| 91 | isNSFW: CONFIG.INSTANCE.IS_NSFW, | ||
| 92 | defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, | ||
| 93 | defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, | ||
| 94 | customizations: { | ||
| 95 | javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT, | ||
| 96 | css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS | ||
| 97 | } | ||
| 98 | }, | ||
| 99 | search: { | ||
| 100 | remoteUri: { | ||
| 101 | users: CONFIG.SEARCH.REMOTE_URI.USERS, | ||
| 102 | anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS | ||
| 103 | }, | ||
| 104 | searchIndex: { | ||
| 105 | enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, | ||
| 106 | url: CONFIG.SEARCH.SEARCH_INDEX.URL, | ||
| 107 | disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, | ||
| 108 | isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH | ||
| 109 | } | ||
| 110 | }, | ||
| 111 | plugin: { | ||
| 112 | registered: this.getRegisteredPlugins(), | ||
| 113 | registeredExternalAuths: this.getExternalAuthsPlugins(), | ||
| 114 | registeredIdAndPassAuths: this.getIdAndPassAuthPlugins() | ||
| 115 | }, | ||
| 116 | theme: { | ||
| 117 | registered: this.getRegisteredThemes(), | ||
| 118 | default: defaultTheme | ||
| 119 | }, | ||
| 120 | email: { | ||
| 121 | enabled: isEmailEnabled() | ||
| 122 | }, | ||
| 123 | contactForm: { | ||
| 124 | enabled: CONFIG.CONTACT_FORM.ENABLED | ||
| 125 | }, | ||
| 126 | serverVersion: PEERTUBE_VERSION, | ||
| 127 | serverCommit: this.serverCommit, | ||
| 128 | transcoding: { | ||
| 129 | remoteRunners: { | ||
| 130 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
| 131 | }, | ||
| 132 | hls: { | ||
| 133 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED | ||
| 134 | }, | ||
| 135 | web_videos: { | ||
| 136 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED | ||
| 137 | }, | ||
| 138 | enabledResolutions: this.getEnabledResolutions('vod'), | ||
| 139 | profile: CONFIG.TRANSCODING.PROFILE, | ||
| 140 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod') | ||
| 141 | }, | ||
| 142 | live: { | ||
| 143 | enabled: CONFIG.LIVE.ENABLED, | ||
| 144 | |||
| 145 | allowReplay: CONFIG.LIVE.ALLOW_REPLAY, | ||
| 146 | latencySetting: { | ||
| 147 | enabled: CONFIG.LIVE.LATENCY_SETTING.ENABLED | ||
| 148 | }, | ||
| 149 | |||
| 150 | maxDuration: CONFIG.LIVE.MAX_DURATION, | ||
| 151 | maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, | ||
| 152 | maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, | ||
| 153 | |||
| 154 | transcoding: { | ||
| 155 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, | ||
| 156 | remoteRunners: { | ||
| 157 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
| 158 | }, | ||
| 159 | enabledResolutions: this.getEnabledResolutions('live'), | ||
| 160 | profile: CONFIG.LIVE.TRANSCODING.PROFILE, | ||
| 161 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') | ||
| 162 | }, | ||
| 163 | |||
| 164 | rtmp: { | ||
| 165 | port: CONFIG.LIVE.RTMP.PORT | ||
| 166 | } | ||
| 167 | }, | ||
| 168 | videoStudio: { | ||
| 169 | enabled: CONFIG.VIDEO_STUDIO.ENABLED, | ||
| 170 | remoteRunners: { | ||
| 171 | enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED | ||
| 172 | } | ||
| 173 | }, | ||
| 174 | videoFile: { | ||
| 175 | update: { | ||
| 176 | enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED | ||
| 177 | } | ||
| 178 | }, | ||
| 179 | import: { | ||
| 180 | videos: { | ||
| 181 | http: { | ||
| 182 | enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED | ||
| 183 | }, | ||
| 184 | torrent: { | ||
| 185 | enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED | ||
| 186 | } | ||
| 187 | }, | ||
| 188 | videoChannelSynchronization: { | ||
| 189 | enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED | ||
| 190 | } | ||
| 191 | }, | ||
| 192 | autoBlacklist: { | ||
| 193 | videos: { | ||
| 194 | ofUsers: { | ||
| 195 | enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED | ||
| 196 | } | ||
| 197 | } | ||
| 198 | }, | ||
| 199 | avatar: { | ||
| 200 | file: { | ||
| 201 | size: { | ||
| 202 | max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max | ||
| 203 | }, | ||
| 204 | extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME | ||
| 205 | } | ||
| 206 | }, | ||
| 207 | banner: { | ||
| 208 | file: { | ||
| 209 | size: { | ||
| 210 | max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max | ||
| 211 | }, | ||
| 212 | extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME | ||
| 213 | } | ||
| 214 | }, | ||
| 215 | video: { | ||
| 216 | image: { | ||
| 217 | extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, | ||
| 218 | size: { | ||
| 219 | max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max | ||
| 220 | } | ||
| 221 | }, | ||
| 222 | file: { | ||
| 223 | extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME | ||
| 224 | } | ||
| 225 | }, | ||
| 226 | videoCaption: { | ||
| 227 | file: { | ||
| 228 | size: { | ||
| 229 | max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max | ||
| 230 | }, | ||
| 231 | extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME | ||
| 232 | } | ||
| 233 | }, | ||
| 234 | user: { | ||
| 235 | videoQuota: CONFIG.USER.VIDEO_QUOTA, | ||
| 236 | videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY | ||
| 237 | }, | ||
| 238 | videoChannels: { | ||
| 239 | maxPerUser: CONFIG.VIDEO_CHANNELS.MAX_PER_USER | ||
| 240 | }, | ||
| 241 | trending: { | ||
| 242 | videos: { | ||
| 243 | intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS, | ||
| 244 | algorithms: { | ||
| 245 | enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, | ||
| 246 | default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT | ||
| 247 | } | ||
| 248 | } | ||
| 249 | }, | ||
| 250 | tracker: { | ||
| 251 | enabled: CONFIG.TRACKER.ENABLED | ||
| 252 | }, | ||
| 253 | |||
| 254 | followings: { | ||
| 255 | instance: { | ||
| 256 | autoFollowIndex: { | ||
| 257 | indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL | ||
| 258 | } | ||
| 259 | } | ||
| 260 | }, | ||
| 261 | |||
| 262 | broadcastMessage: { | ||
| 263 | enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, | ||
| 264 | message: CONFIG.BROADCAST_MESSAGE.MESSAGE, | ||
| 265 | level: CONFIG.BROADCAST_MESSAGE.LEVEL, | ||
| 266 | dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE | ||
| 267 | }, | ||
| 268 | |||
| 269 | homepage: { | ||
| 270 | enabled: this.homepageEnabled | ||
| 271 | } | ||
| 272 | } | ||
| 273 | } | ||
| 274 | |||
| 275 | async getServerConfig (ip?: string): Promise<ServerConfig> { | ||
| 276 | const { allowed } = await Hooks.wrapPromiseFun( | ||
| 277 | isSignupAllowed, | ||
| 278 | |||
| 279 | { | ||
| 280 | ip, | ||
| 281 | signupMode: CONFIG.SIGNUP.REQUIRES_APPROVAL | ||
| 282 | ? 'request-registration' | ||
| 283 | : 'direct-registration' | ||
| 284 | }, | ||
| 285 | |||
| 286 | CONFIG.SIGNUP.REQUIRES_APPROVAL | ||
| 287 | ? 'filter:api.user.request-signup.allowed.result' | ||
| 288 | : 'filter:api.user.signup.allowed.result' | ||
| 289 | ) | ||
| 290 | |||
| 291 | const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip) | ||
| 292 | |||
| 293 | const signup = { | ||
| 294 | allowed, | ||
| 295 | allowedForCurrentIP, | ||
| 296 | minimumAge: CONFIG.SIGNUP.MINIMUM_AGE, | ||
| 297 | requiresApproval: CONFIG.SIGNUP.REQUIRES_APPROVAL, | ||
| 298 | requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION | ||
| 299 | } | ||
| 300 | |||
| 301 | const htmlConfig = await this.getHTMLServerConfig() | ||
| 302 | |||
| 303 | return { ...htmlConfig, signup } | ||
| 304 | } | ||
| 305 | |||
| 306 | getRegisteredThemes () { | ||
| 307 | return PluginManager.Instance.getRegisteredThemes() | ||
| 308 | .map(t => ({ | ||
| 309 | npmName: PluginModel.buildNpmName(t.name, t.type), | ||
| 310 | name: t.name, | ||
| 311 | version: t.version, | ||
| 312 | description: t.description, | ||
| 313 | css: t.css, | ||
| 314 | clientScripts: t.clientScripts | ||
| 315 | })) | ||
| 316 | } | ||
| 317 | |||
| 318 | getRegisteredPlugins () { | ||
| 319 | return PluginManager.Instance.getRegisteredPlugins() | ||
| 320 | .map(p => ({ | ||
| 321 | npmName: PluginModel.buildNpmName(p.name, p.type), | ||
| 322 | name: p.name, | ||
| 323 | version: p.version, | ||
| 324 | description: p.description, | ||
| 325 | clientScripts: p.clientScripts | ||
| 326 | })) | ||
| 327 | } | ||
| 328 | |||
| 329 | getEnabledResolutions (type: 'vod' | 'live') { | ||
| 330 | const transcoding = type === 'vod' | ||
| 331 | ? CONFIG.TRANSCODING | ||
| 332 | : CONFIG.LIVE.TRANSCODING | ||
| 333 | |||
| 334 | return Object.keys(transcoding.RESOLUTIONS) | ||
| 335 | .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true) | ||
| 336 | .map(r => parseInt(r, 10)) | ||
| 337 | } | ||
| 338 | |||
| 339 | private getIdAndPassAuthPlugins () { | ||
| 340 | const result: RegisteredIdAndPassAuthConfig[] = [] | ||
| 341 | |||
| 342 | for (const p of PluginManager.Instance.getIdAndPassAuths()) { | ||
| 343 | for (const auth of p.idAndPassAuths) { | ||
| 344 | result.push({ | ||
| 345 | npmName: p.npmName, | ||
| 346 | name: p.name, | ||
| 347 | version: p.version, | ||
| 348 | authName: auth.authName, | ||
| 349 | weight: auth.getWeight() | ||
| 350 | }) | ||
| 351 | } | ||
| 352 | } | ||
| 353 | |||
| 354 | return result | ||
| 355 | } | ||
| 356 | |||
| 357 | private getExternalAuthsPlugins () { | ||
| 358 | const result: RegisteredExternalAuthConfig[] = [] | ||
| 359 | |||
| 360 | for (const p of PluginManager.Instance.getExternalAuths()) { | ||
| 361 | for (const auth of p.externalAuths) { | ||
| 362 | result.push({ | ||
| 363 | npmName: p.npmName, | ||
| 364 | name: p.name, | ||
| 365 | version: p.version, | ||
| 366 | authName: auth.authName, | ||
| 367 | authDisplayName: auth.authDisplayName() | ||
| 368 | }) | ||
| 369 | } | ||
| 370 | } | ||
| 371 | |||
| 372 | return result | ||
| 373 | } | ||
| 374 | |||
| 375 | static get Instance () { | ||
| 376 | return this.instance || (this.instance = new this()) | ||
| 377 | } | ||
| 378 | } | ||
| 379 | |||
| 380 | // --------------------------------------------------------------------------- | ||
| 381 | |||
| 382 | export { | ||
| 383 | ServerConfigManager | ||
| 384 | } | ||
diff --git a/server/lib/signup.ts b/server/lib/signup.ts deleted file mode 100644 index 6702c22cb..000000000 --- a/server/lib/signup.ts +++ /dev/null | |||
| @@ -1,75 +0,0 @@ | |||
| 1 | import { IPv4, IPv6, parse, subnetMatch } from 'ipaddr.js' | ||
| 2 | import { CONFIG } from '../initializers/config' | ||
| 3 | import { UserModel } from '../models/user/user' | ||
| 4 | |||
| 5 | const isCidr = require('is-cidr') | ||
| 6 | |||
| 7 | export type SignupMode = 'direct-registration' | 'request-registration' | ||
| 8 | |||
| 9 | async function isSignupAllowed (options: { | ||
| 10 | signupMode: SignupMode | ||
| 11 | |||
| 12 | ip: string // For plugins | ||
| 13 | body?: any | ||
| 14 | }): Promise<{ allowed: boolean, errorMessage?: string }> { | ||
| 15 | const { signupMode } = options | ||
| 16 | |||
| 17 | if (CONFIG.SIGNUP.ENABLED === false) { | ||
| 18 | return { allowed: false, errorMessage: 'User registration is not allowed' } | ||
| 19 | } | ||
| 20 | |||
| 21 | if (signupMode === 'direct-registration' && CONFIG.SIGNUP.REQUIRES_APPROVAL === true) { | ||
| 22 | return { allowed: false, errorMessage: 'User registration requires approval' } | ||
| 23 | } | ||
| 24 | |||
| 25 | // No limit and signup is enabled | ||
| 26 | if (CONFIG.SIGNUP.LIMIT === -1) { | ||
| 27 | return { allowed: true } | ||
| 28 | } | ||
| 29 | |||
| 30 | const totalUsers = await UserModel.countTotal() | ||
| 31 | |||
| 32 | return { allowed: totalUsers < CONFIG.SIGNUP.LIMIT, errorMessage: 'User limit is reached on this instance' } | ||
| 33 | } | ||
| 34 | |||
| 35 | function isSignupAllowedForCurrentIP (ip: string) { | ||
| 36 | if (!ip) return false | ||
| 37 | |||
| 38 | const addr = parse(ip) | ||
| 39 | const excludeList = [ 'blacklist' ] | ||
| 40 | let matched = '' | ||
| 41 | |||
| 42 | // if there is a valid, non-empty whitelist, we exclude all unknown addresses too | ||
| 43 | if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { | ||
| 44 | excludeList.push('unknown') | ||
| 45 | } | ||
| 46 | |||
| 47 | if (addr.kind() === 'ipv4') { | ||
| 48 | const addrV4 = IPv4.parse(ip) | ||
| 49 | const rangeList = { | ||
| 50 | whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr)) | ||
| 51 | .map(cidr => IPv4.parseCIDR(cidr)), | ||
| 52 | blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr)) | ||
| 53 | .map(cidr => IPv4.parseCIDR(cidr)) | ||
| 54 | } | ||
| 55 | matched = subnetMatch(addrV4, rangeList, 'unknown') | ||
| 56 | } else if (addr.kind() === 'ipv6') { | ||
| 57 | const addrV6 = IPv6.parse(ip) | ||
| 58 | const rangeList = { | ||
| 59 | whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr)) | ||
| 60 | .map(cidr => IPv6.parseCIDR(cidr)), | ||
| 61 | blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr)) | ||
| 62 | .map(cidr => IPv6.parseCIDR(cidr)) | ||
| 63 | } | ||
| 64 | matched = subnetMatch(addrV6, rangeList, 'unknown') | ||
| 65 | } | ||
| 66 | |||
| 67 | return !excludeList.includes(matched) | ||
| 68 | } | ||
| 69 | |||
| 70 | // --------------------------------------------------------------------------- | ||
| 71 | |||
| 72 | export { | ||
| 73 | isSignupAllowed, | ||
| 74 | isSignupAllowedForCurrentIP | ||
| 75 | } | ||
diff --git a/server/lib/stat-manager.ts b/server/lib/stat-manager.ts deleted file mode 100644 index 0516e7f1a..000000000 --- a/server/lib/stat-manager.ts +++ /dev/null | |||
| @@ -1,182 +0,0 @@ | |||
| 1 | import { mapSeries } from 'bluebird' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { ActorFollowModel } from '@server/models/actor/actor-follow' | ||
| 4 | import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' | ||
| 5 | import { UserModel } from '@server/models/user/user' | ||
| 6 | import { VideoModel } from '@server/models/video/video' | ||
| 7 | import { VideoChannelModel } from '@server/models/video/video-channel' | ||
| 8 | import { VideoCommentModel } from '@server/models/video/video-comment' | ||
| 9 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 10 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | ||
| 11 | import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@shared/models' | ||
| 12 | |||
| 13 | class StatsManager { | ||
| 14 | |||
| 15 | private static instance: StatsManager | ||
| 16 | |||
| 17 | private readonly instanceStartDate = new Date() | ||
| 18 | |||
| 19 | private readonly inboxMessages = { | ||
| 20 | processed: 0, | ||
| 21 | errors: 0, | ||
| 22 | successes: 0, | ||
| 23 | waiting: 0, | ||
| 24 | errorsPerType: this.buildAPPerType(), | ||
| 25 | successesPerType: this.buildAPPerType() | ||
| 26 | } | ||
| 27 | |||
| 28 | private constructor () {} | ||
| 29 | |||
| 30 | updateInboxWaiting (inboxMessagesWaiting: number) { | ||
| 31 | this.inboxMessages.waiting = inboxMessagesWaiting | ||
| 32 | } | ||
| 33 | |||
| 34 | addInboxProcessedSuccess (type: ActivityType) { | ||
| 35 | this.inboxMessages.processed++ | ||
| 36 | this.inboxMessages.successes++ | ||
| 37 | this.inboxMessages.successesPerType[type]++ | ||
| 38 | } | ||
| 39 | |||
| 40 | addInboxProcessedError (type: ActivityType) { | ||
| 41 | this.inboxMessages.processed++ | ||
| 42 | this.inboxMessages.errors++ | ||
| 43 | this.inboxMessages.errorsPerType[type]++ | ||
| 44 | } | ||
| 45 | |||
| 46 | async getStats () { | ||
| 47 | const { totalLocalVideos, totalLocalVideoViews, totalVideos } = await VideoModel.getStats() | ||
| 48 | const { totalLocalVideoComments, totalVideoComments } = await VideoCommentModel.getStats() | ||
| 49 | const { totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers } = await UserModel.getStats() | ||
| 50 | const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() | ||
| 51 | const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() | ||
| 52 | const { | ||
| 53 | totalLocalVideoChannels, | ||
| 54 | totalLocalDailyActiveVideoChannels, | ||
| 55 | totalLocalWeeklyActiveVideoChannels, | ||
| 56 | totalLocalMonthlyActiveVideoChannels | ||
| 57 | } = await VideoChannelModel.getStats() | ||
| 58 | const { totalLocalPlaylists } = await VideoPlaylistModel.getStats() | ||
| 59 | |||
| 60 | const videosRedundancyStats = await this.buildRedundancyStats() | ||
| 61 | |||
| 62 | const data: ServerStats = { | ||
| 63 | totalUsers, | ||
| 64 | totalDailyActiveUsers, | ||
| 65 | totalWeeklyActiveUsers, | ||
| 66 | totalMonthlyActiveUsers, | ||
| 67 | |||
| 68 | totalLocalVideos, | ||
| 69 | totalLocalVideoViews, | ||
| 70 | totalLocalVideoComments, | ||
| 71 | totalLocalVideoFilesSize, | ||
| 72 | |||
| 73 | totalVideos, | ||
| 74 | totalVideoComments, | ||
| 75 | |||
| 76 | totalLocalVideoChannels, | ||
| 77 | totalLocalDailyActiveVideoChannels, | ||
| 78 | totalLocalWeeklyActiveVideoChannels, | ||
| 79 | totalLocalMonthlyActiveVideoChannels, | ||
| 80 | |||
| 81 | totalLocalPlaylists, | ||
| 82 | |||
| 83 | totalInstanceFollowers, | ||
| 84 | totalInstanceFollowing, | ||
| 85 | |||
| 86 | videosRedundancy: videosRedundancyStats, | ||
| 87 | |||
| 88 | ...this.buildAPStats() | ||
| 89 | } | ||
| 90 | |||
| 91 | return data | ||
| 92 | } | ||
| 93 | |||
| 94 | private buildActivityPubMessagesProcessedPerSecond () { | ||
| 95 | const now = new Date() | ||
| 96 | const startedSeconds = (now.getTime() - this.instanceStartDate.getTime()) / 1000 | ||
| 97 | |||
| 98 | return this.inboxMessages.processed / startedSeconds | ||
| 99 | } | ||
| 100 | |||
| 101 | private buildRedundancyStats () { | ||
| 102 | const strategies = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES | ||
| 103 | .map(r => ({ | ||
| 104 | strategy: r.strategy as VideoRedundancyStrategyWithManual, | ||
| 105 | size: r.size | ||
| 106 | })) | ||
| 107 | |||
| 108 | strategies.push({ strategy: 'manual', size: null }) | ||
| 109 | |||
| 110 | return mapSeries(strategies, r => { | ||
| 111 | return VideoRedundancyModel.getStats(r.strategy) | ||
| 112 | .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) | ||
| 113 | }) | ||
| 114 | } | ||
| 115 | |||
| 116 | private buildAPPerType () { | ||
| 117 | return { | ||
| 118 | Create: 0, | ||
| 119 | Update: 0, | ||
| 120 | Delete: 0, | ||
| 121 | Follow: 0, | ||
| 122 | Accept: 0, | ||
| 123 | Reject: 0, | ||
| 124 | Announce: 0, | ||
| 125 | Undo: 0, | ||
| 126 | Like: 0, | ||
| 127 | Dislike: 0, | ||
| 128 | Flag: 0, | ||
| 129 | View: 0 | ||
| 130 | } | ||
| 131 | } | ||
| 132 | |||
| 133 | private buildAPStats () { | ||
| 134 | return { | ||
| 135 | totalActivityPubMessagesProcessed: this.inboxMessages.processed, | ||
| 136 | |||
| 137 | totalActivityPubMessagesSuccesses: this.inboxMessages.successes, | ||
| 138 | |||
| 139 | // Dirty, but simpler and with type checking | ||
| 140 | totalActivityPubCreateMessagesSuccesses: this.inboxMessages.successesPerType.Create, | ||
| 141 | totalActivityPubUpdateMessagesSuccesses: this.inboxMessages.successesPerType.Update, | ||
| 142 | totalActivityPubDeleteMessagesSuccesses: this.inboxMessages.successesPerType.Delete, | ||
| 143 | totalActivityPubFollowMessagesSuccesses: this.inboxMessages.successesPerType.Follow, | ||
| 144 | totalActivityPubAcceptMessagesSuccesses: this.inboxMessages.successesPerType.Accept, | ||
| 145 | totalActivityPubRejectMessagesSuccesses: this.inboxMessages.successesPerType.Reject, | ||
| 146 | totalActivityPubAnnounceMessagesSuccesses: this.inboxMessages.successesPerType.Announce, | ||
| 147 | totalActivityPubUndoMessagesSuccesses: this.inboxMessages.successesPerType.Undo, | ||
| 148 | totalActivityPubLikeMessagesSuccesses: this.inboxMessages.successesPerType.Like, | ||
| 149 | totalActivityPubDislikeMessagesSuccesses: this.inboxMessages.successesPerType.Dislike, | ||
| 150 | totalActivityPubFlagMessagesSuccesses: this.inboxMessages.successesPerType.Flag, | ||
| 151 | totalActivityPubViewMessagesSuccesses: this.inboxMessages.successesPerType.View, | ||
| 152 | |||
| 153 | totalActivityPubCreateMessagesErrors: this.inboxMessages.errorsPerType.Create, | ||
| 154 | totalActivityPubUpdateMessagesErrors: this.inboxMessages.errorsPerType.Update, | ||
| 155 | totalActivityPubDeleteMessagesErrors: this.inboxMessages.errorsPerType.Delete, | ||
| 156 | totalActivityPubFollowMessagesErrors: this.inboxMessages.errorsPerType.Follow, | ||
| 157 | totalActivityPubAcceptMessagesErrors: this.inboxMessages.errorsPerType.Accept, | ||
| 158 | totalActivityPubRejectMessagesErrors: this.inboxMessages.errorsPerType.Reject, | ||
| 159 | totalActivityPubAnnounceMessagesErrors: this.inboxMessages.errorsPerType.Announce, | ||
| 160 | totalActivityPubUndoMessagesErrors: this.inboxMessages.errorsPerType.Undo, | ||
| 161 | totalActivityPubLikeMessagesErrors: this.inboxMessages.errorsPerType.Like, | ||
| 162 | totalActivityPubDislikeMessagesErrors: this.inboxMessages.errorsPerType.Dislike, | ||
| 163 | totalActivityPubFlagMessagesErrors: this.inboxMessages.errorsPerType.Flag, | ||
| 164 | totalActivityPubViewMessagesErrors: this.inboxMessages.errorsPerType.View, | ||
| 165 | |||
| 166 | totalActivityPubMessagesErrors: this.inboxMessages.errors, | ||
| 167 | |||
| 168 | activityPubMessagesProcessedPerSecond: this.buildActivityPubMessagesProcessedPerSecond(), | ||
| 169 | totalActivityPubMessagesWaiting: this.inboxMessages.waiting | ||
| 170 | } | ||
| 171 | } | ||
| 172 | |||
| 173 | static get Instance () { | ||
| 174 | return this.instance || (this.instance = new this()) | ||
| 175 | } | ||
| 176 | } | ||
| 177 | |||
| 178 | // --------------------------------------------------------------------------- | ||
| 179 | |||
| 180 | export { | ||
| 181 | StatsManager | ||
| 182 | } | ||
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts deleted file mode 100644 index 3a805a943..000000000 --- a/server/lib/sync-channel.ts +++ /dev/null | |||
| @@ -1,111 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' | ||
| 3 | import { CONFIG } from '@server/initializers/config' | ||
| 4 | import { buildYoutubeDLImport } from '@server/lib/video-pre-import' | ||
| 5 | import { UserModel } from '@server/models/user/user' | ||
| 6 | import { VideoImportModel } from '@server/models/video/video-import' | ||
| 7 | import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' | ||
| 8 | import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' | ||
| 9 | import { CreateJobArgument, JobQueue } from './job-queue' | ||
| 10 | import { ServerConfigManager } from './server-config-manager' | ||
| 11 | |||
| 12 | export async function synchronizeChannel (options: { | ||
| 13 | channel: MChannelAccountDefault | ||
| 14 | externalChannelUrl: string | ||
| 15 | videosCountLimit: number | ||
| 16 | channelSync?: MChannelSync | ||
| 17 | onlyAfter?: Date | ||
| 18 | }) { | ||
| 19 | const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options | ||
| 20 | |||
| 21 | if (channelSync) { | ||
| 22 | channelSync.state = VideoChannelSyncState.PROCESSING | ||
| 23 | channelSync.lastSyncAt = new Date() | ||
| 24 | await channelSync.save() | ||
| 25 | } | ||
| 26 | |||
| 27 | try { | ||
| 28 | const user = await UserModel.loadByChannelActorId(channel.actorId) | ||
| 29 | const youtubeDL = new YoutubeDLWrapper( | ||
| 30 | externalChannelUrl, | ||
| 31 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
| 32 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
| 33 | ) | ||
| 34 | |||
| 35 | const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) | ||
| 36 | |||
| 37 | logger.info( | ||
| 38 | 'Fetched %d candidate URLs for sync channel %s.', | ||
| 39 | targetUrls.length, channel.Actor.preferredUsername, { targetUrls } | ||
| 40 | ) | ||
| 41 | |||
| 42 | if (targetUrls.length === 0) { | ||
| 43 | if (channelSync) { | ||
| 44 | channelSync.state = VideoChannelSyncState.SYNCED | ||
| 45 | await channelSync.save() | ||
| 46 | } | ||
| 47 | |||
| 48 | return | ||
| 49 | } | ||
| 50 | |||
| 51 | const children: CreateJobArgument[] = [] | ||
| 52 | |||
| 53 | for (const targetUrl of targetUrls) { | ||
| 54 | if (await skipImport(channel, targetUrl, onlyAfter)) continue | ||
| 55 | |||
| 56 | const { job } = await buildYoutubeDLImport({ | ||
| 57 | user, | ||
| 58 | channel, | ||
| 59 | targetUrl, | ||
| 60 | channelSync, | ||
| 61 | importDataOverride: { | ||
| 62 | privacy: VideoPrivacy.PUBLIC | ||
| 63 | } | ||
| 64 | }) | ||
| 65 | |||
| 66 | children.push(job) | ||
| 67 | } | ||
| 68 | |||
| 69 | // Will update the channel sync status | ||
| 70 | const parent: CreateJobArgument = { | ||
| 71 | type: 'after-video-channel-import', | ||
| 72 | payload: { | ||
| 73 | channelSyncId: channelSync?.id | ||
| 74 | } | ||
| 75 | } | ||
| 76 | |||
| 77 | await JobQueue.Instance.createJobWithChildren(parent, children) | ||
| 78 | } catch (err) { | ||
| 79 | logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err }) | ||
| 80 | channelSync.state = VideoChannelSyncState.FAILED | ||
| 81 | await channelSync.save() | ||
| 82 | } | ||
| 83 | } | ||
| 84 | |||
| 85 | // --------------------------------------------------------------------------- | ||
| 86 | |||
| 87 | async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) { | ||
| 88 | if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { | ||
| 89 | logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name) | ||
| 90 | return true | ||
| 91 | } | ||
| 92 | |||
| 93 | if (onlyAfter) { | ||
| 94 | const youtubeDL = new YoutubeDLWrapper( | ||
| 95 | targetUrl, | ||
| 96 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
| 97 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
| 98 | ) | ||
| 99 | |||
| 100 | const videoInfo = await youtubeDL.getInfoForDownload() | ||
| 101 | |||
| 102 | const onlyAfterWithoutTime = new Date(onlyAfter) | ||
| 103 | onlyAfterWithoutTime.setHours(0, 0, 0, 0) | ||
| 104 | |||
| 105 | if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) { | ||
| 106 | return true | ||
| 107 | } | ||
| 108 | } | ||
| 109 | |||
| 110 | return false | ||
| 111 | } | ||
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts deleted file mode 100644 index 0b98da14f..000000000 --- a/server/lib/thumbnail.ts +++ /dev/null | |||
| @@ -1,327 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import { ThumbnailType } from '@shared/models' | ||
| 3 | import { generateImageFilename, generateImageFromVideoFile } from '../helpers/image-utils' | ||
| 4 | import { CONFIG } from '../initializers/config' | ||
| 5 | import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' | ||
| 6 | import { ThumbnailModel } from '../models/video/thumbnail' | ||
| 7 | import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models' | ||
| 8 | import { MThumbnail } from '../types/models/video/thumbnail' | ||
| 9 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' | ||
| 10 | import { VideoPathManager } from './video-path-manager' | ||
| 11 | import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' | ||
| 12 | |||
| 13 | type ImageSize = { height?: number, width?: number } | ||
| 14 | |||
| 15 | function updateLocalPlaylistMiniatureFromExisting (options: { | ||
| 16 | inputPath: string | ||
| 17 | playlist: MVideoPlaylistThumbnail | ||
| 18 | automaticallyGenerated: boolean | ||
| 19 | keepOriginal?: boolean // default to false | ||
| 20 | size?: ImageSize | ||
| 21 | }) { | ||
| 22 | const { inputPath, playlist, automaticallyGenerated, keepOriginal = false, size } = options | ||
| 23 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) | ||
| 24 | const type = ThumbnailType.MINIATURE | ||
| 25 | |||
| 26 | const thumbnailCreator = () => { | ||
| 27 | return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) | ||
| 28 | } | ||
| 29 | |||
| 30 | return updateThumbnailFromFunction({ | ||
| 31 | thumbnailCreator, | ||
| 32 | filename, | ||
| 33 | height, | ||
| 34 | width, | ||
| 35 | type, | ||
| 36 | automaticallyGenerated, | ||
| 37 | onDisk: true, | ||
| 38 | existingThumbnail | ||
| 39 | }) | ||
| 40 | } | ||
| 41 | |||
| 42 | function updateRemotePlaylistMiniatureFromUrl (options: { | ||
| 43 | downloadUrl: string | ||
| 44 | playlist: MVideoPlaylistThumbnail | ||
| 45 | size?: ImageSize | ||
| 46 | }) { | ||
| 47 | const { downloadUrl, playlist, size } = options | ||
| 48 | const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) | ||
| 49 | const type = ThumbnailType.MINIATURE | ||
| 50 | |||
| 51 | // Only save the file URL if it is a remote playlist | ||
| 52 | const fileUrl = playlist.isOwned() | ||
| 53 | ? null | ||
| 54 | : downloadUrl | ||
| 55 | |||
| 56 | const thumbnailCreator = () => { | ||
| 57 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) | ||
| 58 | } | ||
| 59 | |||
| 60 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) | ||
| 61 | } | ||
| 62 | |||
| 63 | function updateLocalVideoMiniatureFromExisting (options: { | ||
| 64 | inputPath: string | ||
| 65 | video: MVideoThumbnail | ||
| 66 | type: ThumbnailType | ||
| 67 | automaticallyGenerated: boolean | ||
| 68 | size?: ImageSize | ||
| 69 | keepOriginal?: boolean // default to false | ||
| 70 | }) { | ||
| 71 | const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options | ||
| 72 | |||
| 73 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
| 74 | |||
| 75 | const thumbnailCreator = () => { | ||
| 76 | return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) | ||
| 77 | } | ||
| 78 | |||
| 79 | return updateThumbnailFromFunction({ | ||
| 80 | thumbnailCreator, | ||
| 81 | filename, | ||
| 82 | height, | ||
| 83 | width, | ||
| 84 | type, | ||
| 85 | automaticallyGenerated, | ||
| 86 | existingThumbnail, | ||
| 87 | onDisk: true | ||
| 88 | }) | ||
| 89 | } | ||
| 90 | |||
| 91 | function generateLocalVideoMiniature (options: { | ||
| 92 | video: MVideoThumbnail | ||
| 93 | videoFile: MVideoFile | ||
| 94 | type: ThumbnailType | ||
| 95 | }) { | ||
| 96 | const { video, videoFile, type } = options | ||
| 97 | |||
| 98 | return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => { | ||
| 99 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | ||
| 100 | |||
| 101 | const thumbnailCreator = videoFile.isAudio() | ||
| 102 | ? () => processImageFromWorker({ | ||
| 103 | path: ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, | ||
| 104 | destination: outputPath, | ||
| 105 | newSize: { width, height }, | ||
| 106 | keepOriginal: true | ||
| 107 | }) | ||
| 108 | : () => generateImageFromVideoFile({ | ||
| 109 | fromPath: input, | ||
| 110 | folder: basePath, | ||
| 111 | imageName: filename, | ||
| 112 | size: { height, width } | ||
| 113 | }) | ||
| 114 | |||
| 115 | return updateThumbnailFromFunction({ | ||
| 116 | thumbnailCreator, | ||
| 117 | filename, | ||
| 118 | height, | ||
| 119 | width, | ||
| 120 | type, | ||
| 121 | automaticallyGenerated: true, | ||
| 122 | onDisk: true, | ||
| 123 | existingThumbnail | ||
| 124 | }) | ||
| 125 | }) | ||
| 126 | } | ||
| 127 | |||
| 128 | // --------------------------------------------------------------------------- | ||
| 129 | |||
| 130 | function updateLocalVideoMiniatureFromUrl (options: { | ||
| 131 | downloadUrl: string | ||
| 132 | video: MVideoThumbnail | ||
| 133 | type: ThumbnailType | ||
| 134 | size?: ImageSize | ||
| 135 | }) { | ||
| 136 | const { downloadUrl, video, type, size } = options | ||
| 137 | const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
| 138 | |||
| 139 | // Only save the file URL if it is a remote video | ||
| 140 | const fileUrl = video.isOwned() | ||
| 141 | ? null | ||
| 142 | : downloadUrl | ||
| 143 | |||
| 144 | const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) | ||
| 145 | |||
| 146 | // Do not change the thumbnail filename if the file did not change | ||
| 147 | const filename = thumbnailUrlChanged | ||
| 148 | ? updatedFilename | ||
| 149 | : existingThumbnail.filename | ||
| 150 | |||
| 151 | const thumbnailCreator = () => { | ||
| 152 | if (thumbnailUrlChanged) { | ||
| 153 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) | ||
| 154 | } | ||
| 155 | |||
| 156 | return Promise.resolve() | ||
| 157 | } | ||
| 158 | |||
| 159 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) | ||
| 160 | } | ||
| 161 | |||
| 162 | function updateRemoteVideoThumbnail (options: { | ||
| 163 | fileUrl: string | ||
| 164 | video: MVideoThumbnail | ||
| 165 | type: ThumbnailType | ||
| 166 | size: ImageSize | ||
| 167 | onDisk: boolean | ||
| 168 | }) { | ||
| 169 | const { fileUrl, video, type, size, onDisk } = options | ||
| 170 | const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
| 171 | |||
| 172 | const thumbnail = existingThumbnail || new ThumbnailModel() | ||
| 173 | |||
| 174 | // Do not change the thumbnail filename if the file did not change | ||
| 175 | if (hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)) { | ||
| 176 | thumbnail.filename = generatedFilename | ||
| 177 | } | ||
| 178 | |||
| 179 | thumbnail.height = height | ||
| 180 | thumbnail.width = width | ||
| 181 | thumbnail.type = type | ||
| 182 | thumbnail.fileUrl = fileUrl | ||
| 183 | thumbnail.onDisk = onDisk | ||
| 184 | |||
| 185 | return thumbnail | ||
| 186 | } | ||
| 187 | |||
| 188 | // --------------------------------------------------------------------------- | ||
| 189 | |||
| 190 | async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) { | ||
| 191 | if (video.getMiniature().automaticallyGenerated === true) { | ||
| 192 | const miniature = await generateLocalVideoMiniature({ | ||
| 193 | video, | ||
| 194 | videoFile: video.getMaxQualityFile(), | ||
| 195 | type: ThumbnailType.MINIATURE | ||
| 196 | }) | ||
| 197 | await video.addAndSaveThumbnail(miniature) | ||
| 198 | } | ||
| 199 | |||
| 200 | if (video.getPreview().automaticallyGenerated === true) { | ||
| 201 | const preview = await generateLocalVideoMiniature({ | ||
| 202 | video, | ||
| 203 | videoFile: video.getMaxQualityFile(), | ||
| 204 | type: ThumbnailType.PREVIEW | ||
| 205 | }) | ||
| 206 | await video.addAndSaveThumbnail(preview) | ||
| 207 | } | ||
| 208 | } | ||
| 209 | |||
| 210 | // --------------------------------------------------------------------------- | ||
| 211 | |||
| 212 | export { | ||
| 213 | generateLocalVideoMiniature, | ||
| 214 | regenerateMiniaturesIfNeeded, | ||
| 215 | updateLocalVideoMiniatureFromUrl, | ||
| 216 | updateLocalVideoMiniatureFromExisting, | ||
| 217 | updateRemoteVideoThumbnail, | ||
| 218 | updateRemotePlaylistMiniatureFromUrl, | ||
| 219 | updateLocalPlaylistMiniatureFromExisting | ||
| 220 | } | ||
| 221 | |||
| 222 | // --------------------------------------------------------------------------- | ||
| 223 | // Private | ||
| 224 | // --------------------------------------------------------------------------- | ||
| 225 | |||
| 226 | function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { | ||
| 227 | const existingUrl = existingThumbnail | ||
| 228 | ? existingThumbnail.fileUrl | ||
| 229 | : null | ||
| 230 | |||
| 231 | // If the thumbnail URL did not change and has a unique filename (introduced in 3.1), avoid thumbnail processing | ||
| 232 | return !existingUrl || existingUrl !== downloadUrl || downloadUrl.endsWith(`${video.uuid}.jpg`) | ||
| 233 | } | ||
| 234 | |||
| 235 | function buildMetadataFromPlaylist (playlist: MVideoPlaylistThumbnail, size: ImageSize) { | ||
| 236 | const filename = playlist.generateThumbnailName() | ||
| 237 | const basePath = CONFIG.STORAGE.THUMBNAILS_DIR | ||
| 238 | |||
| 239 | return { | ||
| 240 | filename, | ||
| 241 | basePath, | ||
| 242 | existingThumbnail: playlist.Thumbnail, | ||
| 243 | outputPath: join(basePath, filename), | ||
| 244 | height: size ? size.height : THUMBNAILS_SIZE.height, | ||
| 245 | width: size ? size.width : THUMBNAILS_SIZE.width | ||
| 246 | } | ||
| 247 | } | ||
| 248 | |||
| 249 | function buildMetadataFromVideo (video: MVideoThumbnail, type: ThumbnailType, size?: ImageSize) { | ||
| 250 | const existingThumbnail = Array.isArray(video.Thumbnails) | ||
| 251 | ? video.Thumbnails.find(t => t.type === type) | ||
| 252 | : undefined | ||
| 253 | |||
| 254 | if (type === ThumbnailType.MINIATURE) { | ||
| 255 | const filename = generateImageFilename() | ||
| 256 | const basePath = CONFIG.STORAGE.THUMBNAILS_DIR | ||
| 257 | |||
| 258 | return { | ||
| 259 | filename, | ||
| 260 | basePath, | ||
| 261 | existingThumbnail, | ||
| 262 | outputPath: join(basePath, filename), | ||
| 263 | height: size ? size.height : THUMBNAILS_SIZE.height, | ||
| 264 | width: size ? size.width : THUMBNAILS_SIZE.width | ||
| 265 | } | ||
| 266 | } | ||
| 267 | |||
| 268 | if (type === ThumbnailType.PREVIEW) { | ||
| 269 | const filename = generateImageFilename() | ||
| 270 | const basePath = CONFIG.STORAGE.PREVIEWS_DIR | ||
| 271 | |||
| 272 | return { | ||
| 273 | filename, | ||
| 274 | basePath, | ||
| 275 | existingThumbnail, | ||
| 276 | outputPath: join(basePath, filename), | ||
| 277 | height: size ? size.height : PREVIEWS_SIZE.height, | ||
| 278 | width: size ? size.width : PREVIEWS_SIZE.width | ||
| 279 | } | ||
| 280 | } | ||
| 281 | |||
| 282 | return undefined | ||
| 283 | } | ||
| 284 | |||
| 285 | async function updateThumbnailFromFunction (parameters: { | ||
| 286 | thumbnailCreator: () => Promise<any> | ||
| 287 | filename: string | ||
| 288 | height: number | ||
| 289 | width: number | ||
| 290 | type: ThumbnailType | ||
| 291 | onDisk: boolean | ||
| 292 | automaticallyGenerated?: boolean | ||
| 293 | fileUrl?: string | ||
| 294 | existingThumbnail?: MThumbnail | ||
| 295 | }) { | ||
| 296 | const { | ||
| 297 | thumbnailCreator, | ||
| 298 | filename, | ||
| 299 | width, | ||
| 300 | height, | ||
| 301 | type, | ||
| 302 | existingThumbnail, | ||
| 303 | onDisk, | ||
| 304 | automaticallyGenerated = null, | ||
| 305 | fileUrl = null | ||
| 306 | } = parameters | ||
| 307 | |||
| 308 | const oldFilename = existingThumbnail && existingThumbnail.filename !== filename | ||
| 309 | ? existingThumbnail.filename | ||
| 310 | : undefined | ||
| 311 | |||
| 312 | const thumbnail: MThumbnail = existingThumbnail || new ThumbnailModel() | ||
| 313 | |||
| 314 | thumbnail.filename = filename | ||
| 315 | thumbnail.height = height | ||
| 316 | thumbnail.width = width | ||
| 317 | thumbnail.type = type | ||
| 318 | thumbnail.fileUrl = fileUrl | ||
| 319 | thumbnail.automaticallyGenerated = automaticallyGenerated | ||
| 320 | thumbnail.onDisk = onDisk | ||
| 321 | |||
| 322 | if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename | ||
| 323 | |||
| 324 | await thumbnailCreator() | ||
| 325 | |||
| 326 | return thumbnail | ||
| 327 | } | ||
diff --git a/server/lib/timeserie.ts b/server/lib/timeserie.ts deleted file mode 100644 index 08b12129a..000000000 --- a/server/lib/timeserie.ts +++ /dev/null | |||
| @@ -1,61 +0,0 @@ | |||
| 1 | import { logger } from '@server/helpers/logger' | ||
| 2 | |||
| 3 | function buildGroupByAndBoundaries (startDateString: string, endDateString: string) { | ||
| 4 | const startDate = new Date(startDateString) | ||
| 5 | const endDate = new Date(endDateString) | ||
| 6 | |||
| 7 | const groupInterval = buildGroupInterval(startDate, endDate) | ||
| 8 | |||
| 9 | logger.debug('Found "%s" group interval.', groupInterval, { startDate, endDate }) | ||
| 10 | |||
| 11 | // Remove parts of the date we don't need | ||
| 12 | if (groupInterval.endsWith(' month') || groupInterval.endsWith(' months')) { | ||
| 13 | startDate.setDate(1) | ||
| 14 | startDate.setHours(0, 0, 0, 0) | ||
| 15 | } else if (groupInterval.endsWith(' day') || groupInterval.endsWith(' days')) { | ||
| 16 | startDate.setHours(0, 0, 0, 0) | ||
| 17 | } else if (groupInterval.endsWith(' hour') || groupInterval.endsWith(' hours')) { | ||
| 18 | startDate.setMinutes(0, 0, 0) | ||
| 19 | } else { | ||
| 20 | startDate.setSeconds(0, 0) | ||
| 21 | } | ||
| 22 | |||
| 23 | return { | ||
| 24 | groupInterval, | ||
| 25 | startDate, | ||
| 26 | endDate | ||
| 27 | } | ||
| 28 | } | ||
| 29 | |||
| 30 | // --------------------------------------------------------------------------- | ||
| 31 | |||
| 32 | export { | ||
| 33 | buildGroupByAndBoundaries | ||
| 34 | } | ||
| 35 | |||
| 36 | // --------------------------------------------------------------------------- | ||
| 37 | |||
| 38 | function buildGroupInterval (startDate: Date, endDate: Date): string { | ||
| 39 | const aYear = 31536000 | ||
| 40 | const aMonth = 2678400 | ||
| 41 | const aDay = 86400 | ||
| 42 | const anHour = 3600 | ||
| 43 | const aMinute = 60 | ||
| 44 | |||
| 45 | const diffSeconds = (endDate.getTime() - startDate.getTime()) / 1000 | ||
| 46 | |||
| 47 | if (diffSeconds >= 6 * aYear) return '6 months' | ||
| 48 | if (diffSeconds >= 2 * aYear) return '1 month' | ||
| 49 | if (diffSeconds >= 6 * aMonth) return '7 days' | ||
| 50 | if (diffSeconds >= 2 * aMonth) return '2 days' | ||
| 51 | |||
| 52 | if (diffSeconds >= 15 * aDay) return '1 day' | ||
| 53 | if (diffSeconds >= 8 * aDay) return '12 hours' | ||
| 54 | if (diffSeconds >= 4 * aDay) return '6 hours' | ||
| 55 | |||
| 56 | if (diffSeconds >= 15 * anHour) return '1 hour' | ||
| 57 | |||
| 58 | if (diffSeconds >= 180 * aMinute) return '10 minutes' | ||
| 59 | |||
| 60 | return '1 minute' | ||
| 61 | } | ||
diff --git a/server/lib/transcoding/create-transcoding-job.ts b/server/lib/transcoding/create-transcoding-job.ts deleted file mode 100644 index d78e68b87..000000000 --- a/server/lib/transcoding/create-transcoding-job.ts +++ /dev/null | |||
| @@ -1,37 +0,0 @@ | |||
| 1 | import { CONFIG } from '@server/initializers/config' | ||
| 2 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
| 3 | import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared' | ||
| 4 | |||
| 5 | export function createOptimizeOrMergeAudioJobs (options: { | ||
| 6 | video: MVideoFullLight | ||
| 7 | videoFile: MVideoFile | ||
| 8 | isNewVideo: boolean | ||
| 9 | user: MUserId | ||
| 10 | videoFileAlreadyLocked: boolean | ||
| 11 | }) { | ||
| 12 | return getJobBuilder().createOptimizeOrMergeAudioJobs(options) | ||
| 13 | } | ||
| 14 | |||
| 15 | // --------------------------------------------------------------------------- | ||
| 16 | |||
| 17 | export function createTranscodingJobs (options: { | ||
| 18 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 | ||
| 19 | video: MVideoFullLight | ||
| 20 | resolutions: number[] | ||
| 21 | isNewVideo: boolean | ||
| 22 | user: MUserId | ||
| 23 | }) { | ||
| 24 | return getJobBuilder().createTranscodingJobs(options) | ||
| 25 | } | ||
| 26 | |||
| 27 | // --------------------------------------------------------------------------- | ||
| 28 | // Private | ||
| 29 | // --------------------------------------------------------------------------- | ||
| 30 | |||
| 31 | function getJobBuilder () { | ||
| 32 | if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) { | ||
| 33 | return new TranscodingRunnerJobBuilder() | ||
| 34 | } | ||
| 35 | |||
| 36 | return new TranscodingJobQueueBuilder() | ||
| 37 | } | ||
diff --git a/server/lib/transcoding/default-transcoding-profiles.ts b/server/lib/transcoding/default-transcoding-profiles.ts deleted file mode 100644 index 8f8fdd026..000000000 --- a/server/lib/transcoding/default-transcoding-profiles.ts +++ /dev/null | |||
| @@ -1,143 +0,0 @@ | |||
| 1 | |||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { FFmpegCommandWrapper, getDefaultAvailableEncoders } from '@shared/ffmpeg' | ||
| 4 | import { AvailableEncoders, EncoderOptionsBuilder } from '@shared/models' | ||
| 5 | |||
| 6 | // --------------------------------------------------------------------------- | ||
| 7 | // Profile manager to get and change default profiles | ||
| 8 | // --------------------------------------------------------------------------- | ||
| 9 | |||
| 10 | class VideoTranscodingProfilesManager { | ||
| 11 | private static instance: VideoTranscodingProfilesManager | ||
| 12 | |||
| 13 | // 1 === less priority | ||
| 14 | private readonly encodersPriorities = { | ||
| 15 | vod: this.buildDefaultEncodersPriorities(), | ||
| 16 | live: this.buildDefaultEncodersPriorities() | ||
| 17 | } | ||
| 18 | |||
| 19 | private readonly availableEncoders = getDefaultAvailableEncoders() | ||
| 20 | |||
| 21 | private availableProfiles = { | ||
| 22 | vod: [] as string[], | ||
| 23 | live: [] as string[] | ||
| 24 | } | ||
| 25 | |||
| 26 | private constructor () { | ||
| 27 | this.buildAvailableProfiles() | ||
| 28 | } | ||
| 29 | |||
| 30 | getAvailableEncoders (): AvailableEncoders { | ||
| 31 | return { | ||
| 32 | available: this.availableEncoders, | ||
| 33 | encodersToTry: { | ||
| 34 | vod: { | ||
| 35 | video: this.getEncodersByPriority('vod', 'video'), | ||
| 36 | audio: this.getEncodersByPriority('vod', 'audio') | ||
| 37 | }, | ||
| 38 | live: { | ||
| 39 | video: this.getEncodersByPriority('live', 'video'), | ||
| 40 | audio: this.getEncodersByPriority('live', 'audio') | ||
| 41 | } | ||
| 42 | } | ||
| 43 | } | ||
| 44 | } | ||
| 45 | |||
| 46 | getAvailableProfiles (type: 'vod' | 'live') { | ||
| 47 | return this.availableProfiles[type] | ||
| 48 | } | ||
| 49 | |||
| 50 | addProfile (options: { | ||
| 51 | type: 'vod' | 'live' | ||
| 52 | encoder: string | ||
| 53 | profile: string | ||
| 54 | builder: EncoderOptionsBuilder | ||
| 55 | }) { | ||
| 56 | const { type, encoder, profile, builder } = options | ||
| 57 | |||
| 58 | const encoders = this.availableEncoders[type] | ||
| 59 | |||
| 60 | if (!encoders[encoder]) encoders[encoder] = {} | ||
| 61 | encoders[encoder][profile] = builder | ||
| 62 | |||
| 63 | this.buildAvailableProfiles() | ||
| 64 | } | ||
| 65 | |||
| 66 | removeProfile (options: { | ||
| 67 | type: 'vod' | 'live' | ||
| 68 | encoder: string | ||
| 69 | profile: string | ||
| 70 | }) { | ||
| 71 | const { type, encoder, profile } = options | ||
| 72 | |||
| 73 | delete this.availableEncoders[type][encoder][profile] | ||
| 74 | this.buildAvailableProfiles() | ||
| 75 | } | ||
| 76 | |||
| 77 | addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { | ||
| 78 | this.encodersPriorities[type][streamType].push({ name: encoder, priority }) | ||
| 79 | |||
| 80 | FFmpegCommandWrapper.resetSupportedEncoders() | ||
| 81 | } | ||
| 82 | |||
| 83 | removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { | ||
| 84 | this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType] | ||
| 85 | .filter(o => o.name !== encoder && o.priority !== priority) | ||
| 86 | |||
| 87 | FFmpegCommandWrapper.resetSupportedEncoders() | ||
| 88 | } | ||
| 89 | |||
| 90 | private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') { | ||
| 91 | return this.encodersPriorities[type][streamType] | ||
| 92 | .sort((e1, e2) => { | ||
| 93 | if (e1.priority > e2.priority) return -1 | ||
| 94 | else if (e1.priority === e2.priority) return 0 | ||
| 95 | |||
| 96 | return 1 | ||
| 97 | }) | ||
| 98 | .map(e => e.name) | ||
| 99 | } | ||
| 100 | |||
| 101 | private buildAvailableProfiles () { | ||
| 102 | for (const type of [ 'vod', 'live' ]) { | ||
| 103 | const result = new Set() | ||
| 104 | |||
| 105 | const encoders = this.availableEncoders[type] | ||
| 106 | |||
| 107 | for (const encoderName of Object.keys(encoders)) { | ||
| 108 | for (const profile of Object.keys(encoders[encoderName])) { | ||
| 109 | result.add(profile) | ||
| 110 | } | ||
| 111 | } | ||
| 112 | |||
| 113 | this.availableProfiles[type] = Array.from(result) | ||
| 114 | } | ||
| 115 | |||
| 116 | logger.debug('Available transcoding profiles built.', { availableProfiles: this.availableProfiles }) | ||
| 117 | } | ||
| 118 | |||
| 119 | private buildDefaultEncodersPriorities () { | ||
| 120 | return { | ||
| 121 | video: [ | ||
| 122 | { name: 'libx264', priority: 100 } | ||
| 123 | ], | ||
| 124 | |||
| 125 | // Try the first one, if not available try the second one etc | ||
| 126 | audio: [ | ||
| 127 | // we favor VBR, if a good AAC encoder is available | ||
| 128 | { name: 'libfdk_aac', priority: 200 }, | ||
| 129 | { name: 'aac', priority: 100 } | ||
| 130 | ] | ||
| 131 | } | ||
| 132 | } | ||
| 133 | |||
| 134 | static get Instance () { | ||
| 135 | return this.instance || (this.instance = new this()) | ||
| 136 | } | ||
| 137 | } | ||
| 138 | |||
| 139 | // --------------------------------------------------------------------------- | ||
| 140 | |||
| 141 | export { | ||
| 142 | VideoTranscodingProfilesManager | ||
| 143 | } | ||
diff --git a/server/lib/transcoding/ended-transcoding.ts b/server/lib/transcoding/ended-transcoding.ts deleted file mode 100644 index d31674ede..000000000 --- a/server/lib/transcoding/ended-transcoding.ts +++ /dev/null | |||
| @@ -1,18 +0,0 @@ | |||
| 1 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 2 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 3 | import { MVideo } from '@server/types/models' | ||
| 4 | import { moveToNextState } from '../video-state' | ||
| 5 | |||
| 6 | export async function onTranscodingEnded (options: { | ||
| 7 | video: MVideo | ||
| 8 | isNewVideo: boolean | ||
| 9 | moveVideoToNextState: boolean | ||
| 10 | }) { | ||
| 11 | const { video, isNewVideo, moveVideoToNextState } = options | ||
| 12 | |||
| 13 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
| 14 | |||
| 15 | if (moveVideoToNextState) { | ||
| 16 | await retryTransactionWrapper(moveToNextState, { video, isNewVideo }) | ||
| 17 | } | ||
| 18 | } | ||
diff --git a/server/lib/transcoding/hls-transcoding.ts b/server/lib/transcoding/hls-transcoding.ts deleted file mode 100644 index 2c325d9ee..000000000 --- a/server/lib/transcoding/hls-transcoding.ts +++ /dev/null | |||
| @@ -1,180 +0,0 @@ | |||
| 1 | import { MutexInterface } from 'async-mutex' | ||
| 2 | import { Job } from 'bullmq' | ||
| 3 | import { ensureDir, move, stat } from 'fs-extra' | ||
| 4 | import { basename, extname as extnameUtil, join } from 'path' | ||
| 5 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
| 7 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 8 | import { MVideo, MVideoFile } from '@server/types/models' | ||
| 9 | import { pick } from '@shared/core-utils' | ||
| 10 | import { getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' | ||
| 11 | import { VideoResolution } from '@shared/models' | ||
| 12 | import { CONFIG } from '../../initializers/config' | ||
| 13 | import { VideoFileModel } from '../../models/video/video-file' | ||
| 14 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
| 15 | import { updatePlaylistAfterFileChange } from '../hls' | ||
| 16 | import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' | ||
| 17 | import { buildFileMetadata } from '../video-file' | ||
| 18 | import { VideoPathManager } from '../video-path-manager' | ||
| 19 | import { buildFFmpegVOD } from './shared' | ||
| 20 | |||
| 21 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | ||
| 22 | export async function generateHlsPlaylistResolutionFromTS (options: { | ||
| 23 | video: MVideo | ||
| 24 | concatenatedTsFilePath: string | ||
| 25 | resolution: VideoResolution | ||
| 26 | fps: number | ||
| 27 | isAAC: boolean | ||
| 28 | inputFileMutexReleaser: MutexInterface.Releaser | ||
| 29 | }) { | ||
| 30 | return generateHlsPlaylistCommon({ | ||
| 31 | type: 'hls-from-ts' as 'hls-from-ts', | ||
| 32 | inputPath: options.concatenatedTsFilePath, | ||
| 33 | |||
| 34 | ...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ]) | ||
| 35 | }) | ||
| 36 | } | ||
| 37 | |||
| 38 | // Generate an HLS playlist from an input file, and update the master playlist | ||
| 39 | export function generateHlsPlaylistResolution (options: { | ||
| 40 | video: MVideo | ||
| 41 | videoInputPath: string | ||
| 42 | resolution: VideoResolution | ||
| 43 | fps: number | ||
| 44 | copyCodecs: boolean | ||
| 45 | inputFileMutexReleaser: MutexInterface.Releaser | ||
| 46 | job?: Job | ||
| 47 | }) { | ||
| 48 | return generateHlsPlaylistCommon({ | ||
| 49 | type: 'hls' as 'hls', | ||
| 50 | inputPath: options.videoInputPath, | ||
| 51 | |||
| 52 | ...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) | ||
| 53 | }) | ||
| 54 | } | ||
| 55 | |||
| 56 | export async function onHLSVideoFileTranscoding (options: { | ||
| 57 | video: MVideo | ||
| 58 | videoFile: MVideoFile | ||
| 59 | videoOutputPath: string | ||
| 60 | m3u8OutputPath: string | ||
| 61 | }) { | ||
| 62 | const { video, videoFile, videoOutputPath, m3u8OutputPath } = options | ||
| 63 | |||
| 64 | // Create or update the playlist | ||
| 65 | const playlist = await retryTransactionWrapper(() => { | ||
| 66 | return sequelizeTypescript.transaction(async transaction => { | ||
| 67 | return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) | ||
| 68 | }) | ||
| 69 | }) | ||
| 70 | videoFile.videoStreamingPlaylistId = playlist.id | ||
| 71 | |||
| 72 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 73 | |||
| 74 | try { | ||
| 75 | await video.reload() | ||
| 76 | |||
| 77 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile) | ||
| 78 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) | ||
| 79 | |||
| 80 | // Move playlist file | ||
| 81 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath)) | ||
| 82 | await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true }) | ||
| 83 | // Move video file | ||
| 84 | await move(videoOutputPath, videoFilePath, { overwrite: true }) | ||
| 85 | |||
| 86 | // Update video duration if it was not set (in case of a live for example) | ||
| 87 | if (!video.duration) { | ||
| 88 | video.duration = await getVideoStreamDuration(videoFilePath) | ||
| 89 | await video.save() | ||
| 90 | } | ||
| 91 | |||
| 92 | const stats = await stat(videoFilePath) | ||
| 93 | |||
| 94 | videoFile.size = stats.size | ||
| 95 | videoFile.fps = await getVideoStreamFPS(videoFilePath) | ||
| 96 | videoFile.metadata = await buildFileMetadata(videoFilePath) | ||
| 97 | |||
| 98 | await createTorrentAndSetInfoHash(playlist, videoFile) | ||
| 99 | |||
| 100 | const oldFile = await VideoFileModel.loadHLSFile({ | ||
| 101 | playlistId: playlist.id, | ||
| 102 | fps: videoFile.fps, | ||
| 103 | resolution: videoFile.resolution | ||
| 104 | }) | ||
| 105 | |||
| 106 | if (oldFile) { | ||
| 107 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | ||
| 108 | await oldFile.destroy() | ||
| 109 | } | ||
| 110 | |||
| 111 | const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined) | ||
| 112 | |||
| 113 | await updatePlaylistAfterFileChange(video, playlist) | ||
| 114 | |||
| 115 | return { resolutionPlaylistPath, videoFile: savedVideoFile } | ||
| 116 | } finally { | ||
| 117 | mutexReleaser() | ||
| 118 | } | ||
| 119 | } | ||
| 120 | |||
| 121 | // --------------------------------------------------------------------------- | ||
| 122 | |||
| 123 | async function generateHlsPlaylistCommon (options: { | ||
| 124 | type: 'hls' | 'hls-from-ts' | ||
| 125 | video: MVideo | ||
| 126 | inputPath: string | ||
| 127 | |||
| 128 | resolution: VideoResolution | ||
| 129 | fps: number | ||
| 130 | |||
| 131 | inputFileMutexReleaser: MutexInterface.Releaser | ||
| 132 | |||
| 133 | copyCodecs?: boolean | ||
| 134 | isAAC?: boolean | ||
| 135 | |||
| 136 | job?: Job | ||
| 137 | }) { | ||
| 138 | const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options | ||
| 139 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
| 140 | |||
| 141 | const videoTranscodedBasePath = join(transcodeDirectory, type) | ||
| 142 | await ensureDir(videoTranscodedBasePath) | ||
| 143 | |||
| 144 | const videoFilename = generateHLSVideoFilename(resolution) | ||
| 145 | const videoOutputPath = join(videoTranscodedBasePath, videoFilename) | ||
| 146 | |||
| 147 | const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) | ||
| 148 | const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename) | ||
| 149 | |||
| 150 | const transcodeOptions = { | ||
| 151 | type, | ||
| 152 | |||
| 153 | inputPath, | ||
| 154 | outputPath: m3u8OutputPath, | ||
| 155 | |||
| 156 | resolution, | ||
| 157 | fps, | ||
| 158 | copyCodecs, | ||
| 159 | |||
| 160 | isAAC, | ||
| 161 | |||
| 162 | inputFileMutexReleaser, | ||
| 163 | |||
| 164 | hlsPlaylist: { | ||
| 165 | videoFilename | ||
| 166 | } | ||
| 167 | } | ||
| 168 | |||
| 169 | await buildFFmpegVOD(job).transcode(transcodeOptions) | ||
| 170 | |||
| 171 | const newVideoFile = new VideoFileModel({ | ||
| 172 | resolution, | ||
| 173 | extname: extnameUtil(videoFilename), | ||
| 174 | size: 0, | ||
| 175 | filename: videoFilename, | ||
| 176 | fps: -1 | ||
| 177 | }) | ||
| 178 | |||
| 179 | await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath }) | ||
| 180 | } | ||
diff --git a/server/lib/transcoding/shared/ffmpeg-builder.ts b/server/lib/transcoding/shared/ffmpeg-builder.ts deleted file mode 100644 index 441445ec4..000000000 --- a/server/lib/transcoding/shared/ffmpeg-builder.ts +++ /dev/null | |||
| @@ -1,18 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { FFmpegVOD } from '@shared/ffmpeg' | ||
| 5 | import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles' | ||
| 6 | |||
| 7 | export function buildFFmpegVOD (job?: Job) { | ||
| 8 | return new FFmpegVOD({ | ||
| 9 | ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()), | ||
| 10 | |||
| 11 | updateJobProgress: progress => { | ||
| 12 | if (!job) return | ||
| 13 | |||
| 14 | job.updateProgress(progress) | ||
| 15 | .catch(err => logger.error('Cannot update ffmpeg job progress', { err })) | ||
| 16 | } | ||
| 17 | }) | ||
| 18 | } | ||
diff --git a/server/lib/transcoding/shared/index.ts b/server/lib/transcoding/shared/index.ts deleted file mode 100644 index f0b45bcbb..000000000 --- a/server/lib/transcoding/shared/index.ts +++ /dev/null | |||
| @@ -1,2 +0,0 @@ | |||
| 1 | export * from './job-builders' | ||
| 2 | export * from './ffmpeg-builder' | ||
diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts deleted file mode 100644 index 15fc814ae..000000000 --- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts +++ /dev/null | |||
| @@ -1,21 +0,0 @@ | |||
| 1 | |||
| 2 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
| 3 | |||
| 4 | export abstract class AbstractJobBuilder { | ||
| 5 | |||
| 6 | abstract createOptimizeOrMergeAudioJobs (options: { | ||
| 7 | video: MVideoFullLight | ||
| 8 | videoFile: MVideoFile | ||
| 9 | isNewVideo: boolean | ||
| 10 | user: MUserId | ||
| 11 | videoFileAlreadyLocked: boolean | ||
| 12 | }): Promise<any> | ||
| 13 | |||
| 14 | abstract createTranscodingJobs (options: { | ||
| 15 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 | ||
| 16 | video: MVideoFullLight | ||
| 17 | resolutions: number[] | ||
| 18 | isNewVideo: boolean | ||
| 19 | user: MUserId | null | ||
| 20 | }): Promise<any> | ||
| 21 | } | ||
diff --git a/server/lib/transcoding/shared/job-builders/index.ts b/server/lib/transcoding/shared/job-builders/index.ts deleted file mode 100644 index 9b1c82adf..000000000 --- a/server/lib/transcoding/shared/job-builders/index.ts +++ /dev/null | |||
| @@ -1,2 +0,0 @@ | |||
| 1 | export * from './transcoding-job-queue-builder' | ||
| 2 | export * from './transcoding-runner-job-builder' | ||
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts deleted file mode 100644 index 0505c2b2f..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts +++ /dev/null | |||
| @@ -1,322 +0,0 @@ | |||
| 1 | import Bluebird from 'bluebird' | ||
| 2 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { CONFIG } from '@server/initializers/config' | ||
| 5 | import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
| 6 | import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' | ||
| 7 | import { Hooks } from '@server/lib/plugins/hooks' | ||
| 8 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 10 | import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' | ||
| 11 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' | ||
| 12 | import { | ||
| 13 | HLSTranscodingPayload, | ||
| 14 | MergeAudioTranscodingPayload, | ||
| 15 | NewWebVideoResolutionTranscodingPayload, | ||
| 16 | OptimizeTranscodingPayload, | ||
| 17 | VideoTranscodingPayload | ||
| 18 | } from '@shared/models' | ||
| 19 | import { getTranscodingJobPriority } from '../../transcoding-priority' | ||
| 20 | import { canDoQuickTranscode } from '../../transcoding-quick-transcode' | ||
| 21 | import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions' | ||
| 22 | import { AbstractJobBuilder } from './abstract-job-builder' | ||
| 23 | |||
| 24 | export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | ||
| 25 | |||
| 26 | async createOptimizeOrMergeAudioJobs (options: { | ||
| 27 | video: MVideoFullLight | ||
| 28 | videoFile: MVideoFile | ||
| 29 | isNewVideo: boolean | ||
| 30 | user: MUserId | ||
| 31 | videoFileAlreadyLocked: boolean | ||
| 32 | }) { | ||
| 33 | const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options | ||
| 34 | |||
| 35 | let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload | ||
| 36 | let nextTranscodingSequentialJobPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | ||
| 37 | |||
| 38 | const mutexReleaser = videoFileAlreadyLocked | ||
| 39 | ? () => {} | ||
| 40 | : await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 41 | |||
| 42 | try { | ||
| 43 | await video.reload() | ||
| 44 | await videoFile.reload() | ||
| 45 | |||
| 46 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { | ||
| 47 | const probe = await ffprobePromise(videoFilePath) | ||
| 48 | |||
| 49 | const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) | ||
| 50 | const hasAudio = await hasAudioStream(videoFilePath, probe) | ||
| 51 | const quickTranscode = await canDoQuickTranscode(videoFilePath, probe) | ||
| 52 | const inputFPS = videoFile.isAudio() | ||
| 53 | ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value | ||
| 54 | : await getVideoStreamFPS(videoFilePath, probe) | ||
| 55 | |||
| 56 | const maxResolution = await isAudioFile(videoFilePath, probe) | ||
| 57 | ? DEFAULT_AUDIO_RESOLUTION | ||
| 58 | : buildOriginalFileResolution(resolution) | ||
| 59 | |||
| 60 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { | ||
| 61 | nextTranscodingSequentialJobPayloads.push([ | ||
| 62 | this.buildHLSJobPayload({ | ||
| 63 | deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, | ||
| 64 | |||
| 65 | // We had some issues with a web video quick transcoded while producing a HLS version of it | ||
| 66 | copyCodecs: !quickTranscode, | ||
| 67 | |||
| 68 | resolution: maxResolution, | ||
| 69 | fps: computeOutputFPS({ inputFPS, resolution: maxResolution }), | ||
| 70 | videoUUID: video.uuid, | ||
| 71 | isNewVideo | ||
| 72 | }) | ||
| 73 | ]) | ||
| 74 | } | ||
| 75 | |||
| 76 | const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({ | ||
| 77 | video, | ||
| 78 | inputVideoResolution: maxResolution, | ||
| 79 | inputVideoFPS: inputFPS, | ||
| 80 | hasAudio, | ||
| 81 | isNewVideo | ||
| 82 | }) | ||
| 83 | |||
| 84 | nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ] | ||
| 85 | |||
| 86 | const hasChildren = nextTranscodingSequentialJobPayloads.length !== 0 | ||
| 87 | mergeOrOptimizePayload = videoFile.isAudio() | ||
| 88 | ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo, hasChildren }) | ||
| 89 | : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode, hasChildren }) | ||
| 90 | }) | ||
| 91 | } finally { | ||
| 92 | mutexReleaser() | ||
| 93 | } | ||
| 94 | |||
| 95 | const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => { | ||
| 96 | return Bluebird.mapSeries(payloads, payload => { | ||
| 97 | return this.buildTranscodingJob({ payload, user }) | ||
| 98 | }) | ||
| 99 | }) | ||
| 100 | |||
| 101 | const transcodingJobBuilderJob: CreateJobArgument = { | ||
| 102 | type: 'transcoding-job-builder', | ||
| 103 | payload: { | ||
| 104 | videoUUID: video.uuid, | ||
| 105 | sequentialJobs: nextTranscodingSequentialJobs | ||
| 106 | } | ||
| 107 | } | ||
| 108 | |||
| 109 | const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user }) | ||
| 110 | |||
| 111 | await JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ]) | ||
| 112 | |||
| 113 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
| 114 | } | ||
| 115 | |||
| 116 | // --------------------------------------------------------------------------- | ||
| 117 | |||
| 118 | async createTranscodingJobs (options: { | ||
| 119 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 | ||
| 120 | video: MVideoFullLight | ||
| 121 | resolutions: number[] | ||
| 122 | isNewVideo: boolean | ||
| 123 | user: MUserId | null | ||
| 124 | }) { | ||
| 125 | const { video, transcodingType, resolutions, isNewVideo } = options | ||
| 126 | |||
| 127 | const maxResolution = Math.max(...resolutions) | ||
| 128 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | ||
| 129 | |||
| 130 | logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) | ||
| 131 | |||
| 132 | const { fps: inputFPS } = await video.probeMaxQualityFile() | ||
| 133 | |||
| 134 | const children = childrenResolutions.map(resolution => { | ||
| 135 | const fps = computeOutputFPS({ inputFPS, resolution }) | ||
| 136 | |||
| 137 | if (transcodingType === 'hls') { | ||
| 138 | return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | ||
| 139 | } | ||
| 140 | |||
| 141 | if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { | ||
| 142 | return this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | ||
| 143 | } | ||
| 144 | |||
| 145 | throw new Error('Unknown transcoding type') | ||
| 146 | }) | ||
| 147 | |||
| 148 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
| 149 | |||
| 150 | const parent = transcodingType === 'hls' | ||
| 151 | ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | ||
| 152 | : this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | ||
| 153 | |||
| 154 | // Process the last resolution after the other ones to prevent concurrency issue | ||
| 155 | // Because low resolutions use the biggest one as ffmpeg input | ||
| 156 | await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null }) | ||
| 157 | } | ||
| 158 | |||
| 159 | // --------------------------------------------------------------------------- | ||
| 160 | |||
| 161 | private async createTranscodingJobsWithChildren (options: { | ||
| 162 | videoUUID: string | ||
| 163 | parent: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload) | ||
| 164 | children: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)[] | ||
| 165 | user: MUserId | null | ||
| 166 | }) { | ||
| 167 | const { videoUUID, parent, children, user } = options | ||
| 168 | |||
| 169 | const parentJob = await this.buildTranscodingJob({ payload: parent, user }) | ||
| 170 | const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user })) | ||
| 171 | |||
| 172 | await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs) | ||
| 173 | |||
| 174 | await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length) | ||
| 175 | } | ||
| 176 | |||
| 177 | private async buildTranscodingJob (options: { | ||
| 178 | payload: VideoTranscodingPayload | ||
| 179 | user: MUserId | null // null means we don't want priority | ||
| 180 | }) { | ||
| 181 | const { user, payload } = options | ||
| 182 | |||
| 183 | return { | ||
| 184 | type: 'video-transcoding' as 'video-transcoding', | ||
| 185 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }), | ||
| 186 | payload | ||
| 187 | } | ||
| 188 | } | ||
| 189 | |||
| 190 | private async buildLowerResolutionJobPayloads (options: { | ||
| 191 | video: MVideoWithFileThumbnail | ||
| 192 | inputVideoResolution: number | ||
| 193 | inputVideoFPS: number | ||
| 194 | hasAudio: boolean | ||
| 195 | isNewVideo: boolean | ||
| 196 | }) { | ||
| 197 | const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options | ||
| 198 | |||
| 199 | // Create transcoding jobs if there are enabled resolutions | ||
| 200 | const resolutionsEnabled = await Hooks.wrapObject( | ||
| 201 | computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), | ||
| 202 | 'filter:transcoding.auto.resolutions-to-transcode.result', | ||
| 203 | options | ||
| 204 | ) | ||
| 205 | |||
| 206 | const sequentialPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | ||
| 207 | |||
| 208 | for (const resolution of resolutionsEnabled) { | ||
| 209 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) | ||
| 210 | |||
| 211 | if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { | ||
| 212 | const payloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ | ||
| 213 | this.buildWebVideoJobPayload({ | ||
| 214 | videoUUID: video.uuid, | ||
| 215 | resolution, | ||
| 216 | fps, | ||
| 217 | isNewVideo | ||
| 218 | }) | ||
| 219 | ] | ||
| 220 | |||
| 221 | // Create a subsequent job to create HLS resolution that will just copy web video codecs | ||
| 222 | if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
| 223 | payloads.push( | ||
| 224 | this.buildHLSJobPayload({ | ||
| 225 | videoUUID: video.uuid, | ||
| 226 | resolution, | ||
| 227 | fps, | ||
| 228 | isNewVideo, | ||
| 229 | copyCodecs: true | ||
| 230 | }) | ||
| 231 | ) | ||
| 232 | } | ||
| 233 | |||
| 234 | sequentialPayloads.push(payloads) | ||
| 235 | } else if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
| 236 | sequentialPayloads.push([ | ||
| 237 | this.buildHLSJobPayload({ | ||
| 238 | videoUUID: video.uuid, | ||
| 239 | resolution, | ||
| 240 | fps, | ||
| 241 | copyCodecs: false, | ||
| 242 | isNewVideo | ||
| 243 | }) | ||
| 244 | ]) | ||
| 245 | } | ||
| 246 | } | ||
| 247 | |||
| 248 | return sequentialPayloads | ||
| 249 | } | ||
| 250 | |||
| 251 | private buildHLSJobPayload (options: { | ||
| 252 | videoUUID: string | ||
| 253 | resolution: number | ||
| 254 | fps: number | ||
| 255 | isNewVideo: boolean | ||
| 256 | deleteWebVideoFiles?: boolean // default false | ||
| 257 | copyCodecs?: boolean // default false | ||
| 258 | }): HLSTranscodingPayload { | ||
| 259 | const { videoUUID, resolution, fps, isNewVideo, deleteWebVideoFiles = false, copyCodecs = false } = options | ||
| 260 | |||
| 261 | return { | ||
| 262 | type: 'new-resolution-to-hls', | ||
| 263 | videoUUID, | ||
| 264 | resolution, | ||
| 265 | fps, | ||
| 266 | copyCodecs, | ||
| 267 | isNewVideo, | ||
| 268 | deleteWebVideoFiles | ||
| 269 | } | ||
| 270 | } | ||
| 271 | |||
| 272 | private buildWebVideoJobPayload (options: { | ||
| 273 | videoUUID: string | ||
| 274 | resolution: number | ||
| 275 | fps: number | ||
| 276 | isNewVideo: boolean | ||
| 277 | }): NewWebVideoResolutionTranscodingPayload { | ||
| 278 | const { videoUUID, resolution, fps, isNewVideo } = options | ||
| 279 | |||
| 280 | return { | ||
| 281 | type: 'new-resolution-to-web-video', | ||
| 282 | videoUUID, | ||
| 283 | isNewVideo, | ||
| 284 | resolution, | ||
| 285 | fps | ||
| 286 | } | ||
| 287 | } | ||
| 288 | |||
| 289 | private buildMergeAudioPayload (options: { | ||
| 290 | videoUUID: string | ||
| 291 | isNewVideo: boolean | ||
| 292 | hasChildren: boolean | ||
| 293 | }): MergeAudioTranscodingPayload { | ||
| 294 | const { videoUUID, isNewVideo, hasChildren } = options | ||
| 295 | |||
| 296 | return { | ||
| 297 | type: 'merge-audio-to-web-video', | ||
| 298 | resolution: DEFAULT_AUDIO_RESOLUTION, | ||
| 299 | fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, | ||
| 300 | videoUUID, | ||
| 301 | isNewVideo, | ||
| 302 | hasChildren | ||
| 303 | } | ||
| 304 | } | ||
| 305 | |||
| 306 | private buildOptimizePayload (options: { | ||
| 307 | videoUUID: string | ||
| 308 | quickTranscode: boolean | ||
| 309 | isNewVideo: boolean | ||
| 310 | hasChildren: boolean | ||
| 311 | }): OptimizeTranscodingPayload { | ||
| 312 | const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options | ||
| 313 | |||
| 314 | return { | ||
| 315 | type: 'optimize-to-web-video', | ||
| 316 | videoUUID, | ||
| 317 | isNewVideo, | ||
| 318 | hasChildren, | ||
| 319 | quickTranscode | ||
| 320 | } | ||
| 321 | } | ||
| 322 | } | ||
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts deleted file mode 100644 index f0671bd7a..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ /dev/null | |||
| @@ -1,196 +0,0 @@ | |||
| 1 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
| 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 3 | import { CONFIG } from '@server/initializers/config' | ||
| 4 | import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
| 5 | import { Hooks } from '@server/lib/plugins/hooks' | ||
| 6 | import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners' | ||
| 7 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
| 8 | import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' | ||
| 9 | import { MRunnerJob } from '@server/types/models/runners' | ||
| 10 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' | ||
| 11 | import { getTranscodingJobPriority } from '../../transcoding-priority' | ||
| 12 | import { computeResolutionsToTranscode } from '../../transcoding-resolutions' | ||
| 13 | import { AbstractJobBuilder } from './abstract-job-builder' | ||
| 14 | |||
| 15 | /** | ||
| 16 | * | ||
| 17 | * Class to build transcoding job in the local job queue | ||
| 18 | * | ||
| 19 | */ | ||
| 20 | |||
| 21 | const lTags = loggerTagsFactory('transcoding') | ||
| 22 | |||
| 23 | export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | ||
| 24 | |||
| 25 | async createOptimizeOrMergeAudioJobs (options: { | ||
| 26 | video: MVideoFullLight | ||
| 27 | videoFile: MVideoFile | ||
| 28 | isNewVideo: boolean | ||
| 29 | user: MUserId | ||
| 30 | videoFileAlreadyLocked: boolean | ||
| 31 | }) { | ||
| 32 | const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options | ||
| 33 | |||
| 34 | const mutexReleaser = videoFileAlreadyLocked | ||
| 35 | ? () => {} | ||
| 36 | : await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 37 | |||
| 38 | try { | ||
| 39 | await video.reload() | ||
| 40 | await videoFile.reload() | ||
| 41 | |||
| 42 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { | ||
| 43 | const probe = await ffprobePromise(videoFilePath) | ||
| 44 | |||
| 45 | const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) | ||
| 46 | const hasAudio = await hasAudioStream(videoFilePath, probe) | ||
| 47 | const inputFPS = videoFile.isAudio() | ||
| 48 | ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value | ||
| 49 | : await getVideoStreamFPS(videoFilePath, probe) | ||
| 50 | |||
| 51 | const maxResolution = await isAudioFile(videoFilePath, probe) | ||
| 52 | ? DEFAULT_AUDIO_RESOLUTION | ||
| 53 | : resolution | ||
| 54 | |||
| 55 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
| 56 | const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) | ||
| 57 | |||
| 58 | const mainRunnerJob = videoFile.isAudio() | ||
| 59 | ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) | ||
| 60 | : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) | ||
| 61 | |||
| 62 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { | ||
| 63 | await new VODHLSTranscodingJobHandler().create({ | ||
| 64 | video, | ||
| 65 | deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, | ||
| 66 | resolution: maxResolution, | ||
| 67 | fps, | ||
| 68 | isNewVideo, | ||
| 69 | dependsOnRunnerJob: mainRunnerJob, | ||
| 70 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) | ||
| 71 | }) | ||
| 72 | } | ||
| 73 | |||
| 74 | await this.buildLowerResolutionJobPayloads({ | ||
| 75 | video, | ||
| 76 | inputVideoResolution: maxResolution, | ||
| 77 | inputVideoFPS: inputFPS, | ||
| 78 | hasAudio, | ||
| 79 | isNewVideo, | ||
| 80 | mainRunnerJob, | ||
| 81 | user | ||
| 82 | }) | ||
| 83 | }) | ||
| 84 | } finally { | ||
| 85 | mutexReleaser() | ||
| 86 | } | ||
| 87 | } | ||
| 88 | |||
| 89 | // --------------------------------------------------------------------------- | ||
| 90 | |||
| 91 | async createTranscodingJobs (options: { | ||
| 92 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 | ||
| 93 | video: MVideoFullLight | ||
| 94 | resolutions: number[] | ||
| 95 | isNewVideo: boolean | ||
| 96 | user: MUserId | null | ||
| 97 | }) { | ||
| 98 | const { video, transcodingType, resolutions, isNewVideo, user } = options | ||
| 99 | |||
| 100 | const maxResolution = Math.max(...resolutions) | ||
| 101 | const { fps: inputFPS } = await video.probeMaxQualityFile() | ||
| 102 | const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
| 103 | const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) | ||
| 104 | |||
| 105 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | ||
| 106 | |||
| 107 | logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) | ||
| 108 | |||
| 109 | // Process the last resolution before the other ones to prevent concurrency issue | ||
| 110 | // Because low resolutions use the biggest one as ffmpeg input | ||
| 111 | const mainJob = transcodingType === 'hls' | ||
| 112 | // eslint-disable-next-line max-len | ||
| 113 | ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) | ||
| 114 | : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) | ||
| 115 | |||
| 116 | for (const resolution of childrenResolutions) { | ||
| 117 | const dependsOnRunnerJob = mainJob | ||
| 118 | const fps = computeOutputFPS({ inputFPS, resolution }) | ||
| 119 | |||
| 120 | if (transcodingType === 'hls') { | ||
| 121 | await new VODHLSTranscodingJobHandler().create({ | ||
| 122 | video, | ||
| 123 | resolution, | ||
| 124 | fps, | ||
| 125 | isNewVideo, | ||
| 126 | deleteWebVideoFiles: false, | ||
| 127 | dependsOnRunnerJob, | ||
| 128 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) | ||
| 129 | }) | ||
| 130 | continue | ||
| 131 | } | ||
| 132 | |||
| 133 | if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { | ||
| 134 | await new VODWebVideoTranscodingJobHandler().create({ | ||
| 135 | video, | ||
| 136 | resolution, | ||
| 137 | fps, | ||
| 138 | isNewVideo, | ||
| 139 | dependsOnRunnerJob, | ||
| 140 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) | ||
| 141 | }) | ||
| 142 | continue | ||
| 143 | } | ||
| 144 | |||
| 145 | throw new Error('Unknown transcoding type') | ||
| 146 | } | ||
| 147 | } | ||
| 148 | |||
| 149 | private async buildLowerResolutionJobPayloads (options: { | ||
| 150 | mainRunnerJob: MRunnerJob | ||
| 151 | video: MVideoWithFileThumbnail | ||
| 152 | inputVideoResolution: number | ||
| 153 | inputVideoFPS: number | ||
| 154 | hasAudio: boolean | ||
| 155 | isNewVideo: boolean | ||
| 156 | user: MUserId | ||
| 157 | }) { | ||
| 158 | const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options | ||
| 159 | |||
| 160 | // Create transcoding jobs if there are enabled resolutions | ||
| 161 | const resolutionsEnabled = await Hooks.wrapObject( | ||
| 162 | computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), | ||
| 163 | 'filter:transcoding.auto.resolutions-to-transcode.result', | ||
| 164 | options | ||
| 165 | ) | ||
| 166 | |||
| 167 | logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) }) | ||
| 168 | |||
| 169 | for (const resolution of resolutionsEnabled) { | ||
| 170 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) | ||
| 171 | |||
| 172 | if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { | ||
| 173 | await new VODWebVideoTranscodingJobHandler().create({ | ||
| 174 | video, | ||
| 175 | resolution, | ||
| 176 | fps, | ||
| 177 | isNewVideo, | ||
| 178 | dependsOnRunnerJob: mainRunnerJob, | ||
| 179 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) | ||
| 180 | }) | ||
| 181 | } | ||
| 182 | |||
| 183 | if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
| 184 | await new VODHLSTranscodingJobHandler().create({ | ||
| 185 | video, | ||
| 186 | resolution, | ||
| 187 | fps, | ||
| 188 | isNewVideo, | ||
| 189 | deleteWebVideoFiles: false, | ||
| 190 | dependsOnRunnerJob: mainRunnerJob, | ||
| 191 | priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) | ||
| 192 | }) | ||
| 193 | } | ||
| 194 | } | ||
| 195 | } | ||
| 196 | } | ||
diff --git a/server/lib/transcoding/transcoding-priority.ts b/server/lib/transcoding/transcoding-priority.ts deleted file mode 100644 index 82ab6f2f1..000000000 --- a/server/lib/transcoding/transcoding-priority.ts +++ /dev/null | |||
| @@ -1,24 +0,0 @@ | |||
| 1 | import { JOB_PRIORITY } from '@server/initializers/constants' | ||
| 2 | import { VideoModel } from '@server/models/video/video' | ||
| 3 | import { MUserId } from '@server/types/models' | ||
| 4 | |||
| 5 | export async function getTranscodingJobPriority (options: { | ||
| 6 | user: MUserId | ||
| 7 | fallback: number | ||
| 8 | type: 'vod' | 'studio' | ||
| 9 | }) { | ||
| 10 | const { user, fallback, type } = options | ||
| 11 | |||
| 12 | if (!user) return fallback | ||
| 13 | |||
| 14 | const now = new Date() | ||
| 15 | const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) | ||
| 16 | |||
| 17 | const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) | ||
| 18 | |||
| 19 | const base = type === 'vod' | ||
| 20 | ? JOB_PRIORITY.TRANSCODING | ||
| 21 | : JOB_PRIORITY.VIDEO_STUDIO | ||
| 22 | |||
| 23 | return base + videoUploadedByUser | ||
| 24 | } | ||
diff --git a/server/lib/transcoding/transcoding-quick-transcode.ts b/server/lib/transcoding/transcoding-quick-transcode.ts deleted file mode 100644 index 53f12cd06..000000000 --- a/server/lib/transcoding/transcoding-quick-transcode.ts +++ /dev/null | |||
| @@ -1,12 +0,0 @@ | |||
| 1 | import { FfprobeData } from 'fluent-ffmpeg' | ||
| 2 | import { CONFIG } from '@server/initializers/config' | ||
| 3 | import { canDoQuickAudioTranscode, canDoQuickVideoTranscode, ffprobePromise } from '@shared/ffmpeg' | ||
| 4 | |||
| 5 | export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise<boolean> { | ||
| 6 | if (CONFIG.TRANSCODING.PROFILE !== 'default') return false | ||
| 7 | |||
| 8 | const probe = existingProbe || await ffprobePromise(path) | ||
| 9 | |||
| 10 | return await canDoQuickVideoTranscode(path, probe) && | ||
| 11 | await canDoQuickAudioTranscode(path, probe) | ||
| 12 | } | ||
diff --git a/server/lib/transcoding/transcoding-resolutions.ts b/server/lib/transcoding/transcoding-resolutions.ts deleted file mode 100644 index 9a6bf5722..000000000 --- a/server/lib/transcoding/transcoding-resolutions.ts +++ /dev/null | |||
| @@ -1,73 +0,0 @@ | |||
| 1 | import { CONFIG } from '@server/initializers/config' | ||
| 2 | import { toEven } from '@shared/core-utils' | ||
| 3 | import { VideoResolution } from '@shared/models' | ||
| 4 | |||
| 5 | export function buildOriginalFileResolution (inputResolution: number) { | ||
| 6 | if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { | ||
| 7 | return toEven(inputResolution) | ||
| 8 | } | ||
| 9 | |||
| 10 | const resolutions = computeResolutionsToTranscode({ | ||
| 11 | input: inputResolution, | ||
| 12 | type: 'vod', | ||
| 13 | includeInput: false, | ||
| 14 | strictLower: false, | ||
| 15 | // We don't really care about the audio resolution in this context | ||
| 16 | hasAudio: true | ||
| 17 | }) | ||
| 18 | |||
| 19 | if (resolutions.length === 0) { | ||
| 20 | return toEven(inputResolution) | ||
| 21 | } | ||
| 22 | |||
| 23 | return Math.max(...resolutions) | ||
| 24 | } | ||
| 25 | |||
| 26 | export function computeResolutionsToTranscode (options: { | ||
| 27 | input: number | ||
| 28 | type: 'vod' | 'live' | ||
| 29 | includeInput: boolean | ||
| 30 | strictLower: boolean | ||
| 31 | hasAudio: boolean | ||
| 32 | }) { | ||
| 33 | const { input, type, includeInput, strictLower, hasAudio } = options | ||
| 34 | |||
| 35 | const configResolutions = type === 'vod' | ||
| 36 | ? CONFIG.TRANSCODING.RESOLUTIONS | ||
| 37 | : CONFIG.LIVE.TRANSCODING.RESOLUTIONS | ||
| 38 | |||
| 39 | const resolutionsEnabled = new Set<number>() | ||
| 40 | |||
| 41 | // Put in the order we want to proceed jobs | ||
| 42 | const availableResolutions: VideoResolution[] = [ | ||
| 43 | VideoResolution.H_NOVIDEO, | ||
| 44 | VideoResolution.H_480P, | ||
| 45 | VideoResolution.H_360P, | ||
| 46 | VideoResolution.H_720P, | ||
| 47 | VideoResolution.H_240P, | ||
| 48 | VideoResolution.H_144P, | ||
| 49 | VideoResolution.H_1080P, | ||
| 50 | VideoResolution.H_1440P, | ||
| 51 | VideoResolution.H_4K | ||
| 52 | ] | ||
| 53 | |||
| 54 | for (const resolution of availableResolutions) { | ||
| 55 | // Resolution not enabled | ||
| 56 | if (configResolutions[resolution + 'p'] !== true) continue | ||
| 57 | // Too big resolution for input file | ||
| 58 | if (input < resolution) continue | ||
| 59 | // We only want lower resolutions than input file | ||
| 60 | if (strictLower && input === resolution) continue | ||
| 61 | // Audio resolutio but no audio in the video | ||
| 62 | if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue | ||
| 63 | |||
| 64 | resolutionsEnabled.add(resolution) | ||
| 65 | } | ||
| 66 | |||
| 67 | if (includeInput) { | ||
| 68 | // Always use an even resolution to avoid issues with ffmpeg | ||
| 69 | resolutionsEnabled.add(toEven(input)) | ||
| 70 | } | ||
| 71 | |||
| 72 | return Array.from(resolutionsEnabled) | ||
| 73 | } | ||
diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts deleted file mode 100644 index f92d457a0..000000000 --- a/server/lib/transcoding/web-transcoding.ts +++ /dev/null | |||
| @@ -1,263 +0,0 @@ | |||
| 1 | import { Job } from 'bullmq' | ||
| 2 | import { copyFile, move, remove, stat } from 'fs-extra' | ||
| 3 | import { basename, join } from 'path' | ||
| 4 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
| 5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
| 6 | import { VideoModel } from '@server/models/video/video' | ||
| 7 | import { MVideoFile, MVideoFullLight } from '@server/types/models' | ||
| 8 | import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@shared/ffmpeg' | ||
| 9 | import { VideoResolution, VideoStorage } from '@shared/models' | ||
| 10 | import { CONFIG } from '../../initializers/config' | ||
| 11 | import { VideoFileModel } from '../../models/video/video-file' | ||
| 12 | import { JobQueue } from '../job-queue' | ||
| 13 | import { generateWebVideoFilename } from '../paths' | ||
| 14 | import { buildFileMetadata } from '../video-file' | ||
| 15 | import { VideoPathManager } from '../video-path-manager' | ||
| 16 | import { buildFFmpegVOD } from './shared' | ||
| 17 | import { buildOriginalFileResolution } from './transcoding-resolutions' | ||
| 18 | |||
| 19 | // Optimize the original video file and replace it. The resolution is not changed. | ||
| 20 | export async function optimizeOriginalVideofile (options: { | ||
| 21 | video: MVideoFullLight | ||
| 22 | inputVideoFile: MVideoFile | ||
| 23 | quickTranscode: boolean | ||
| 24 | job: Job | ||
| 25 | }) { | ||
| 26 | const { video, inputVideoFile, quickTranscode, job } = options | ||
| 27 | |||
| 28 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
| 29 | const newExtname = '.mp4' | ||
| 30 | |||
| 31 | // Will be released by our transcodeVOD function once ffmpeg is ran | ||
| 32 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 33 | |||
| 34 | try { | ||
| 35 | await video.reload() | ||
| 36 | await inputVideoFile.reload() | ||
| 37 | |||
| 38 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | ||
| 39 | |||
| 40 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { | ||
| 41 | const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | ||
| 42 | |||
| 43 | const transcodeType: TranscodeVODOptionsType = quickTranscode | ||
| 44 | ? 'quick-transcode' | ||
| 45 | : 'video' | ||
| 46 | |||
| 47 | const resolution = buildOriginalFileResolution(inputVideoFile.resolution) | ||
| 48 | const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution }) | ||
| 49 | |||
| 50 | // Could be very long! | ||
| 51 | await buildFFmpegVOD(job).transcode({ | ||
| 52 | type: transcodeType, | ||
| 53 | |||
| 54 | inputPath: videoInputPath, | ||
| 55 | outputPath: videoOutputPath, | ||
| 56 | |||
| 57 | inputFileMutexReleaser, | ||
| 58 | |||
| 59 | resolution, | ||
| 60 | fps | ||
| 61 | }) | ||
| 62 | |||
| 63 | // Important to do this before getVideoFilename() to take in account the new filename | ||
| 64 | inputVideoFile.resolution = resolution | ||
| 65 | inputVideoFile.extname = newExtname | ||
| 66 | inputVideoFile.filename = generateWebVideoFilename(resolution, newExtname) | ||
| 67 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
| 68 | |||
| 69 | const { videoFile } = await onWebVideoFileTranscoding({ | ||
| 70 | video, | ||
| 71 | videoFile: inputVideoFile, | ||
| 72 | videoOutputPath | ||
| 73 | }) | ||
| 74 | |||
| 75 | await remove(videoInputPath) | ||
| 76 | |||
| 77 | return { transcodeType, videoFile } | ||
| 78 | }) | ||
| 79 | |||
| 80 | return result | ||
| 81 | } finally { | ||
| 82 | inputFileMutexReleaser() | ||
| 83 | } | ||
| 84 | } | ||
| 85 | |||
| 86 | // Transcode the original video file to a lower resolution compatible with web browsers | ||
| 87 | export async function transcodeNewWebVideoResolution (options: { | ||
| 88 | video: MVideoFullLight | ||
| 89 | resolution: VideoResolution | ||
| 90 | fps: number | ||
| 91 | job: Job | ||
| 92 | }) { | ||
| 93 | const { video: videoArg, resolution, fps, job } = options | ||
| 94 | |||
| 95 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
| 96 | const newExtname = '.mp4' | ||
| 97 | |||
| 98 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) | ||
| 99 | |||
| 100 | try { | ||
| 101 | const video = await VideoModel.loadFull(videoArg.uuid) | ||
| 102 | const file = video.getMaxQualityFile().withVideoOrPlaylist(video) | ||
| 103 | |||
| 104 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { | ||
| 105 | const newVideoFile = new VideoFileModel({ | ||
| 106 | resolution, | ||
| 107 | extname: newExtname, | ||
| 108 | filename: generateWebVideoFilename(resolution, newExtname), | ||
| 109 | size: 0, | ||
| 110 | videoId: video.id | ||
| 111 | }) | ||
| 112 | |||
| 113 | const videoOutputPath = join(transcodeDirectory, newVideoFile.filename) | ||
| 114 | |||
| 115 | const transcodeOptions = { | ||
| 116 | type: 'video' as 'video', | ||
| 117 | |||
| 118 | inputPath: videoInputPath, | ||
| 119 | outputPath: videoOutputPath, | ||
| 120 | |||
| 121 | inputFileMutexReleaser, | ||
| 122 | |||
| 123 | resolution, | ||
| 124 | fps | ||
| 125 | } | ||
| 126 | |||
| 127 | await buildFFmpegVOD(job).transcode(transcodeOptions) | ||
| 128 | |||
| 129 | return onWebVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) | ||
| 130 | }) | ||
| 131 | |||
| 132 | return result | ||
| 133 | } finally { | ||
| 134 | inputFileMutexReleaser() | ||
| 135 | } | ||
| 136 | } | ||
| 137 | |||
| 138 | // Merge an image with an audio file to create a video | ||
| 139 | export async function mergeAudioVideofile (options: { | ||
| 140 | video: MVideoFullLight | ||
| 141 | resolution: VideoResolution | ||
| 142 | fps: number | ||
| 143 | job: Job | ||
| 144 | }) { | ||
| 145 | const { video: videoArg, resolution, fps, job } = options | ||
| 146 | |||
| 147 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
| 148 | const newExtname = '.mp4' | ||
| 149 | |||
| 150 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) | ||
| 151 | |||
| 152 | try { | ||
| 153 | const video = await VideoModel.loadFull(videoArg.uuid) | ||
| 154 | const inputVideoFile = video.getMinQualityFile() | ||
| 155 | |||
| 156 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | ||
| 157 | |||
| 158 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { | ||
| 159 | const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | ||
| 160 | |||
| 161 | // If the user updates the video preview during transcoding | ||
| 162 | const previewPath = video.getPreview().getPath() | ||
| 163 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | ||
| 164 | await copyFile(previewPath, tmpPreviewPath) | ||
| 165 | |||
| 166 | const transcodeOptions = { | ||
| 167 | type: 'merge-audio' as 'merge-audio', | ||
| 168 | |||
| 169 | inputPath: tmpPreviewPath, | ||
| 170 | outputPath: videoOutputPath, | ||
| 171 | |||
| 172 | inputFileMutexReleaser, | ||
| 173 | |||
| 174 | audioPath: audioInputPath, | ||
| 175 | resolution, | ||
| 176 | fps | ||
| 177 | } | ||
| 178 | |||
| 179 | try { | ||
| 180 | await buildFFmpegVOD(job).transcode(transcodeOptions) | ||
| 181 | |||
| 182 | await remove(audioInputPath) | ||
| 183 | await remove(tmpPreviewPath) | ||
| 184 | } catch (err) { | ||
| 185 | await remove(tmpPreviewPath) | ||
| 186 | throw err | ||
| 187 | } | ||
| 188 | |||
| 189 | // Important to do this before getVideoFilename() to take in account the new file extension | ||
| 190 | inputVideoFile.extname = newExtname | ||
| 191 | inputVideoFile.resolution = resolution | ||
| 192 | inputVideoFile.filename = generateWebVideoFilename(inputVideoFile.resolution, newExtname) | ||
| 193 | |||
| 194 | // ffmpeg generated a new video file, so update the video duration | ||
| 195 | // See https://trac.ffmpeg.org/ticket/5456 | ||
| 196 | video.duration = await getVideoStreamDuration(videoOutputPath) | ||
| 197 | await video.save() | ||
| 198 | |||
| 199 | return onWebVideoFileTranscoding({ | ||
| 200 | video, | ||
| 201 | videoFile: inputVideoFile, | ||
| 202 | videoOutputPath, | ||
| 203 | wasAudioFile: true | ||
| 204 | }) | ||
| 205 | }) | ||
| 206 | |||
| 207 | return result | ||
| 208 | } finally { | ||
| 209 | inputFileMutexReleaser() | ||
| 210 | } | ||
| 211 | } | ||
| 212 | |||
| 213 | export async function onWebVideoFileTranscoding (options: { | ||
| 214 | video: MVideoFullLight | ||
| 215 | videoFile: MVideoFile | ||
| 216 | videoOutputPath: string | ||
| 217 | wasAudioFile?: boolean // default false | ||
| 218 | }) { | ||
| 219 | const { video, videoFile, videoOutputPath, wasAudioFile } = options | ||
| 220 | |||
| 221 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 222 | |||
| 223 | try { | ||
| 224 | await video.reload() | ||
| 225 | |||
| 226 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) | ||
| 227 | |||
| 228 | const stats = await stat(videoOutputPath) | ||
| 229 | |||
| 230 | const probe = await ffprobePromise(videoOutputPath) | ||
| 231 | const fps = await getVideoStreamFPS(videoOutputPath, probe) | ||
| 232 | const metadata = await buildFileMetadata(videoOutputPath, probe) | ||
| 233 | |||
| 234 | await move(videoOutputPath, outputPath, { overwrite: true }) | ||
| 235 | |||
| 236 | videoFile.size = stats.size | ||
| 237 | videoFile.fps = fps | ||
| 238 | videoFile.metadata = metadata | ||
| 239 | |||
| 240 | await createTorrentAndSetInfoHash(video, videoFile) | ||
| 241 | |||
| 242 | const oldFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | ||
| 243 | if (oldFile) await video.removeWebVideoFile(oldFile) | ||
| 244 | |||
| 245 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | ||
| 246 | video.VideoFiles = await video.$get('VideoFiles') | ||
| 247 | |||
| 248 | if (wasAudioFile) { | ||
| 249 | await JobQueue.Instance.createJob({ | ||
| 250 | type: 'generate-video-storyboard' as 'generate-video-storyboard', | ||
| 251 | payload: { | ||
| 252 | videoUUID: video.uuid, | ||
| 253 | // No need to federate, we process these jobs sequentially | ||
| 254 | federate: false | ||
| 255 | } | ||
| 256 | }) | ||
| 257 | } | ||
| 258 | |||
| 259 | return { video, videoFile } | ||
| 260 | } finally { | ||
| 261 | mutexReleaser() | ||
| 262 | } | ||
| 263 | } | ||
diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts deleted file mode 100644 index c7e0eb414..000000000 --- a/server/lib/uploadx.ts +++ /dev/null | |||
| @@ -1,37 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { buildLogger } from '@server/helpers/logger' | ||
| 3 | import { getResumableUploadPath } from '@server/helpers/upload' | ||
| 4 | import { CONFIG } from '@server/initializers/config' | ||
| 5 | import { LogLevel, Uploadx } from '@uploadx/core' | ||
| 6 | import { extname } from 'path' | ||
| 7 | |||
| 8 | const logger = buildLogger('uploadx') | ||
| 9 | |||
| 10 | const uploadx = new Uploadx({ | ||
| 11 | directory: getResumableUploadPath(), | ||
| 12 | |||
| 13 | expiration: { maxAge: undefined, rolling: true }, | ||
| 14 | |||
| 15 | // Could be big with thumbnails/previews | ||
| 16 | maxMetadataSize: '10MB', | ||
| 17 | |||
| 18 | logger: { | ||
| 19 | logLevel: CONFIG.LOG.LEVEL as LogLevel, | ||
| 20 | debug: logger.debug.bind(logger), | ||
| 21 | info: logger.info.bind(logger), | ||
| 22 | warn: logger.warn.bind(logger), | ||
| 23 | error: logger.error.bind(logger) | ||
| 24 | }, | ||
| 25 | |||
| 26 | userIdentifier: (_, res: express.Response) => { | ||
| 27 | if (!res.locals.oauth) return undefined | ||
| 28 | |||
| 29 | return res.locals.oauth.token.user.id + '' | ||
| 30 | }, | ||
| 31 | |||
| 32 | filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}` | ||
| 33 | }) | ||
| 34 | |||
| 35 | export { | ||
| 36 | uploadx | ||
| 37 | } | ||
diff --git a/server/lib/user.ts b/server/lib/user.ts deleted file mode 100644 index 56995cca3..000000000 --- a/server/lib/user.ts +++ /dev/null | |||
| @@ -1,301 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize/types' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { CONFIG } from '@server/initializers/config' | ||
| 4 | import { UserModel } from '@server/models/user/user' | ||
| 5 | import { MActorDefault } from '@server/types/models/actor' | ||
| 6 | import { ActivityPubActorType } from '../../shared/models/activitypub' | ||
| 7 | import { UserAdminFlag, UserNotificationSetting, UserNotificationSettingValue, UserRole } from '../../shared/models/users' | ||
| 8 | import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' | ||
| 9 | import { sequelizeTypescript } from '../initializers/database' | ||
| 10 | import { AccountModel } from '../models/account/account' | ||
| 11 | import { UserNotificationSettingModel } from '../models/user/user-notification-setting' | ||
| 12 | import { MAccountDefault, MChannelActor } from '../types/models' | ||
| 13 | import { MRegistration, MUser, MUserDefault, MUserId } from '../types/models/user' | ||
| 14 | import { generateAndSaveActorKeys } from './activitypub/actors' | ||
| 15 | import { getLocalAccountActivityPubUrl } from './activitypub/url' | ||
| 16 | import { Emailer } from './emailer' | ||
| 17 | import { LiveQuotaStore } from './live/live-quota-store' | ||
| 18 | import { buildActorInstance, findAvailableLocalActorName } from './local-actor' | ||
| 19 | import { Redis } from './redis' | ||
| 20 | import { createLocalVideoChannel } from './video-channel' | ||
| 21 | import { createWatchLaterPlaylist } from './video-playlist' | ||
| 22 | |||
| 23 | type ChannelNames = { name: string, displayName: string } | ||
| 24 | |||
| 25 | function buildUser (options: { | ||
| 26 | username: string | ||
| 27 | password: string | ||
| 28 | email: string | ||
| 29 | |||
| 30 | role?: UserRole // Default to UserRole.User | ||
| 31 | adminFlags?: UserAdminFlag // Default to UserAdminFlag.NONE | ||
| 32 | |||
| 33 | emailVerified: boolean | null | ||
| 34 | |||
| 35 | videoQuota?: number // Default to CONFIG.USER.VIDEO_QUOTA | ||
| 36 | videoQuotaDaily?: number // Default to CONFIG.USER.VIDEO_QUOTA_DAILY | ||
| 37 | |||
| 38 | pluginAuth?: string | ||
| 39 | }): MUser { | ||
| 40 | const { | ||
| 41 | username, | ||
| 42 | password, | ||
| 43 | email, | ||
| 44 | role = UserRole.USER, | ||
| 45 | emailVerified, | ||
| 46 | videoQuota = CONFIG.USER.VIDEO_QUOTA, | ||
| 47 | videoQuotaDaily = CONFIG.USER.VIDEO_QUOTA_DAILY, | ||
| 48 | adminFlags = UserAdminFlag.NONE, | ||
| 49 | pluginAuth | ||
| 50 | } = options | ||
| 51 | |||
| 52 | return new UserModel({ | ||
| 53 | username, | ||
| 54 | password, | ||
| 55 | email, | ||
| 56 | |||
| 57 | nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, | ||
| 58 | p2pEnabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED, | ||
| 59 | videosHistoryEnabled: CONFIG.USER.HISTORY.VIDEOS.ENABLED, | ||
| 60 | |||
| 61 | autoPlayVideo: true, | ||
| 62 | |||
| 63 | role, | ||
| 64 | emailVerified, | ||
| 65 | adminFlags, | ||
| 66 | |||
| 67 | videoQuota, | ||
| 68 | videoQuotaDaily, | ||
| 69 | |||
| 70 | pluginAuth | ||
| 71 | }) | ||
| 72 | } | ||
| 73 | |||
| 74 | // --------------------------------------------------------------------------- | ||
| 75 | |||
| 76 | async function createUserAccountAndChannelAndPlaylist (parameters: { | ||
| 77 | userToCreate: MUser | ||
| 78 | userDisplayName?: string | ||
| 79 | channelNames?: ChannelNames | ||
| 80 | validateUser?: boolean | ||
| 81 | }): Promise<{ user: MUserDefault, account: MAccountDefault, videoChannel: MChannelActor }> { | ||
| 82 | const { userToCreate, userDisplayName, channelNames, validateUser = true } = parameters | ||
| 83 | |||
| 84 | const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { | ||
| 85 | const userOptions = { | ||
| 86 | transaction: t, | ||
| 87 | validate: validateUser | ||
| 88 | } | ||
| 89 | |||
| 90 | const userCreated: MUserDefault = await userToCreate.save(userOptions) | ||
| 91 | userCreated.NotificationSetting = await createDefaultUserNotificationSettings(userCreated, t) | ||
| 92 | |||
| 93 | const accountCreated = await createLocalAccountWithoutKeys({ | ||
| 94 | name: userCreated.username, | ||
| 95 | displayName: userDisplayName, | ||
| 96 | userId: userCreated.id, | ||
| 97 | applicationId: null, | ||
| 98 | t | ||
| 99 | }) | ||
| 100 | userCreated.Account = accountCreated | ||
| 101 | |||
| 102 | const channelAttributes = await buildChannelAttributes({ user: userCreated, transaction: t, channelNames }) | ||
| 103 | const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t) | ||
| 104 | |||
| 105 | const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) | ||
| 106 | |||
| 107 | return { user: userCreated, account: accountCreated, videoChannel, videoPlaylist } | ||
| 108 | }) | ||
| 109 | |||
| 110 | const [ accountActorWithKeys, channelActorWithKeys ] = await Promise.all([ | ||
| 111 | generateAndSaveActorKeys(account.Actor), | ||
| 112 | generateAndSaveActorKeys(videoChannel.Actor) | ||
| 113 | ]) | ||
| 114 | |||
| 115 | account.Actor = accountActorWithKeys | ||
| 116 | videoChannel.Actor = channelActorWithKeys | ||
| 117 | |||
| 118 | return { user, account, videoChannel } | ||
| 119 | } | ||
| 120 | |||
| 121 | async function createLocalAccountWithoutKeys (parameters: { | ||
| 122 | name: string | ||
| 123 | displayName?: string | ||
| 124 | userId: number | null | ||
| 125 | applicationId: number | null | ||
| 126 | t: Transaction | undefined | ||
| 127 | type?: ActivityPubActorType | ||
| 128 | }) { | ||
| 129 | const { name, displayName, userId, applicationId, t, type = 'Person' } = parameters | ||
| 130 | const url = getLocalAccountActivityPubUrl(name) | ||
| 131 | |||
| 132 | const actorInstance = buildActorInstance(type, url, name) | ||
| 133 | const actorInstanceCreated: MActorDefault = await actorInstance.save({ transaction: t }) | ||
| 134 | |||
| 135 | const accountInstance = new AccountModel({ | ||
| 136 | name: displayName || name, | ||
| 137 | userId, | ||
| 138 | applicationId, | ||
| 139 | actorId: actorInstanceCreated.id | ||
| 140 | }) | ||
| 141 | |||
| 142 | const accountInstanceCreated: MAccountDefault = await accountInstance.save({ transaction: t }) | ||
| 143 | accountInstanceCreated.Actor = actorInstanceCreated | ||
| 144 | |||
| 145 | return accountInstanceCreated | ||
| 146 | } | ||
| 147 | |||
| 148 | async function createApplicationActor (applicationId: number) { | ||
| 149 | const accountCreated = await createLocalAccountWithoutKeys({ | ||
| 150 | name: SERVER_ACTOR_NAME, | ||
| 151 | userId: null, | ||
| 152 | applicationId, | ||
| 153 | t: undefined, | ||
| 154 | type: 'Application' | ||
| 155 | }) | ||
| 156 | |||
| 157 | accountCreated.Actor = await generateAndSaveActorKeys(accountCreated.Actor) | ||
| 158 | |||
| 159 | return accountCreated | ||
| 160 | } | ||
| 161 | |||
| 162 | // --------------------------------------------------------------------------- | ||
| 163 | |||
| 164 | async function sendVerifyUserEmail (user: MUser, isPendingEmail = false) { | ||
| 165 | const verificationString = await Redis.Instance.setUserVerifyEmailVerificationString(user.id) | ||
| 166 | let verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?userId=${user.id}&verificationString=${verificationString}` | ||
| 167 | |||
| 168 | if (isPendingEmail) verifyEmailUrl += '&isPendingEmail=true' | ||
| 169 | |||
| 170 | const to = isPendingEmail | ||
| 171 | ? user.pendingEmail | ||
| 172 | : user.email | ||
| 173 | |||
| 174 | const username = user.username | ||
| 175 | |||
| 176 | Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: false }) | ||
| 177 | } | ||
| 178 | |||
| 179 | async function sendVerifyRegistrationEmail (registration: MRegistration) { | ||
| 180 | const verificationString = await Redis.Instance.setRegistrationVerifyEmailVerificationString(registration.id) | ||
| 181 | const verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?registrationId=${registration.id}&verificationString=${verificationString}` | ||
| 182 | |||
| 183 | const to = registration.email | ||
| 184 | const username = registration.username | ||
| 185 | |||
| 186 | Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: true }) | ||
| 187 | } | ||
| 188 | |||
| 189 | // --------------------------------------------------------------------------- | ||
| 190 | |||
| 191 | async function getOriginalVideoFileTotalFromUser (user: MUserId) { | ||
| 192 | // Don't use sequelize because we need to use a sub query | ||
| 193 | const query = UserModel.generateUserQuotaBaseSQL({ | ||
| 194 | withSelect: true, | ||
| 195 | whereUserId: '$userId', | ||
| 196 | daily: false | ||
| 197 | }) | ||
| 198 | |||
| 199 | const base = await UserModel.getTotalRawQuery(query, user.id) | ||
| 200 | |||
| 201 | return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) | ||
| 202 | } | ||
| 203 | |||
| 204 | // Returns cumulative size of all video files uploaded in the last 24 hours. | ||
| 205 | async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { | ||
| 206 | // Don't use sequelize because we need to use a sub query | ||
| 207 | const query = UserModel.generateUserQuotaBaseSQL({ | ||
| 208 | withSelect: true, | ||
| 209 | whereUserId: '$userId', | ||
| 210 | daily: true | ||
| 211 | }) | ||
| 212 | |||
| 213 | const base = await UserModel.getTotalRawQuery(query, user.id) | ||
| 214 | |||
| 215 | return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) | ||
| 216 | } | ||
| 217 | |||
| 218 | async function isAbleToUploadVideo (userId: number, newVideoSize: number) { | ||
| 219 | const user = await UserModel.loadById(userId) | ||
| 220 | |||
| 221 | if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) | ||
| 222 | |||
| 223 | const [ totalBytes, totalBytesDaily ] = await Promise.all([ | ||
| 224 | getOriginalVideoFileTotalFromUser(user), | ||
| 225 | getOriginalVideoFileTotalDailyFromUser(user) | ||
| 226 | ]) | ||
| 227 | |||
| 228 | const uploadedTotal = newVideoSize + totalBytes | ||
| 229 | const uploadedDaily = newVideoSize + totalBytesDaily | ||
| 230 | |||
| 231 | logger.debug( | ||
| 232 | 'Check user %d quota to upload another video.', userId, | ||
| 233 | { totalBytes, totalBytesDaily, videoQuota: user.videoQuota, videoQuotaDaily: user.videoQuotaDaily, newVideoSize } | ||
| 234 | ) | ||
| 235 | |||
| 236 | if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota | ||
| 237 | if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily | ||
| 238 | |||
| 239 | return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily | ||
| 240 | } | ||
| 241 | |||
| 242 | // --------------------------------------------------------------------------- | ||
| 243 | |||
| 244 | export { | ||
| 245 | getOriginalVideoFileTotalFromUser, | ||
| 246 | getOriginalVideoFileTotalDailyFromUser, | ||
| 247 | createApplicationActor, | ||
| 248 | createUserAccountAndChannelAndPlaylist, | ||
| 249 | createLocalAccountWithoutKeys, | ||
| 250 | |||
| 251 | sendVerifyUserEmail, | ||
| 252 | sendVerifyRegistrationEmail, | ||
| 253 | |||
| 254 | isAbleToUploadVideo, | ||
| 255 | buildUser | ||
| 256 | } | ||
| 257 | |||
| 258 | // --------------------------------------------------------------------------- | ||
| 259 | |||
| 260 | function createDefaultUserNotificationSettings (user: MUserId, t: Transaction | undefined) { | ||
| 261 | const values: UserNotificationSetting & { userId: number } = { | ||
| 262 | userId: user.id, | ||
| 263 | newVideoFromSubscription: UserNotificationSettingValue.WEB, | ||
| 264 | newCommentOnMyVideo: UserNotificationSettingValue.WEB, | ||
| 265 | myVideoImportFinished: UserNotificationSettingValue.WEB, | ||
| 266 | myVideoPublished: UserNotificationSettingValue.WEB, | ||
| 267 | abuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
| 268 | videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
| 269 | blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
| 270 | newUserRegistration: UserNotificationSettingValue.WEB, | ||
| 271 | commentMention: UserNotificationSettingValue.WEB, | ||
| 272 | newFollow: UserNotificationSettingValue.WEB, | ||
| 273 | newInstanceFollower: UserNotificationSettingValue.WEB, | ||
| 274 | abuseNewMessage: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
| 275 | abuseStateChange: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
| 276 | autoInstanceFollowing: UserNotificationSettingValue.WEB, | ||
| 277 | newPeerTubeVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
| 278 | newPluginVersion: UserNotificationSettingValue.WEB, | ||
| 279 | myVideoStudioEditionFinished: UserNotificationSettingValue.WEB | ||
| 280 | } | ||
| 281 | |||
| 282 | return UserNotificationSettingModel.create(values, { transaction: t }) | ||
| 283 | } | ||
| 284 | |||
| 285 | async function buildChannelAttributes (options: { | ||
| 286 | user: MUser | ||
| 287 | transaction?: Transaction | ||
| 288 | channelNames?: ChannelNames | ||
| 289 | }) { | ||
| 290 | const { user, transaction, channelNames } = options | ||
| 291 | |||
| 292 | if (channelNames) return channelNames | ||
| 293 | |||
| 294 | const channelName = await findAvailableLocalActorName(user.username + '_channel', transaction) | ||
| 295 | const videoChannelDisplayName = `Main ${user.username} channel` | ||
| 296 | |||
| 297 | return { | ||
| 298 | name: channelName, | ||
| 299 | displayName: videoChannelDisplayName | ||
| 300 | } | ||
| 301 | } | ||
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts deleted file mode 100644 index d5664a1b9..000000000 --- a/server/lib/video-blacklist.ts +++ /dev/null | |||
| @@ -1,145 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { afterCommitIfTransaction } from '@server/helpers/database-utils' | ||
| 3 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 4 | import { | ||
| 5 | MUser, | ||
| 6 | MVideoAccountLight, | ||
| 7 | MVideoBlacklist, | ||
| 8 | MVideoBlacklistVideo, | ||
| 9 | MVideoFullLight, | ||
| 10 | MVideoWithBlacklistLight | ||
| 11 | } from '@server/types/models' | ||
| 12 | import { LiveVideoError, UserRight, VideoBlacklistCreate, VideoBlacklistType } from '../../shared/models' | ||
| 13 | import { UserAdminFlag } from '../../shared/models/users/user-flag.model' | ||
| 14 | import { logger, loggerTagsFactory } from '../helpers/logger' | ||
| 15 | import { CONFIG } from '../initializers/config' | ||
| 16 | import { VideoBlacklistModel } from '../models/video/video-blacklist' | ||
| 17 | import { sendDeleteVideo } from './activitypub/send' | ||
| 18 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
| 19 | import { LiveManager } from './live/live-manager' | ||
| 20 | import { Notifier } from './notifier' | ||
| 21 | import { Hooks } from './plugins/hooks' | ||
| 22 | |||
| 23 | const lTags = loggerTagsFactory('blacklist') | ||
| 24 | |||
| 25 | async function autoBlacklistVideoIfNeeded (parameters: { | ||
| 26 | video: MVideoWithBlacklistLight | ||
| 27 | user?: MUser | ||
| 28 | isRemote: boolean | ||
| 29 | isNew: boolean | ||
| 30 | isNewFile: boolean | ||
| 31 | notify?: boolean | ||
| 32 | transaction?: Transaction | ||
| 33 | }) { | ||
| 34 | const { video, user, isRemote, isNew, isNewFile, notify = true, transaction } = parameters | ||
| 35 | const doAutoBlacklist = await Hooks.wrapFun( | ||
| 36 | autoBlacklistNeeded, | ||
| 37 | { video, user, isRemote, isNew, isNewFile }, | ||
| 38 | 'filter:video.auto-blacklist.result' | ||
| 39 | ) | ||
| 40 | |||
| 41 | if (!doAutoBlacklist) return false | ||
| 42 | |||
| 43 | const videoBlacklistToCreate = { | ||
| 44 | videoId: video.id, | ||
| 45 | unfederated: true, | ||
| 46 | reason: 'Auto-blacklisted. Moderator review required.', | ||
| 47 | type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED | ||
| 48 | } | ||
| 49 | const [ videoBlacklist ] = await VideoBlacklistModel.findOrCreate<MVideoBlacklistVideo>({ | ||
| 50 | where: { | ||
| 51 | videoId: video.id | ||
| 52 | }, | ||
| 53 | defaults: videoBlacklistToCreate, | ||
| 54 | transaction | ||
| 55 | }) | ||
| 56 | video.VideoBlacklist = videoBlacklist | ||
| 57 | |||
| 58 | videoBlacklist.Video = video | ||
| 59 | |||
| 60 | if (notify) { | ||
| 61 | afterCommitIfTransaction(transaction, () => { | ||
| 62 | Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) | ||
| 63 | }) | ||
| 64 | } | ||
| 65 | |||
| 66 | logger.info('Video %s auto-blacklisted.', video.uuid, lTags(video.uuid)) | ||
| 67 | |||
| 68 | return true | ||
| 69 | } | ||
| 70 | |||
| 71 | async function blacklistVideo (videoInstance: MVideoAccountLight, options: VideoBlacklistCreate) { | ||
| 72 | const blacklist: MVideoBlacklistVideo = await VideoBlacklistModel.create({ | ||
| 73 | videoId: videoInstance.id, | ||
| 74 | unfederated: options.unfederate === true, | ||
| 75 | reason: options.reason, | ||
| 76 | type: VideoBlacklistType.MANUAL | ||
| 77 | }) | ||
| 78 | blacklist.Video = videoInstance | ||
| 79 | |||
| 80 | if (options.unfederate === true) { | ||
| 81 | await sendDeleteVideo(videoInstance, undefined) | ||
| 82 | } | ||
| 83 | |||
| 84 | if (videoInstance.isLive) { | ||
| 85 | LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED) | ||
| 86 | } | ||
| 87 | |||
| 88 | Notifier.Instance.notifyOnVideoBlacklist(blacklist) | ||
| 89 | } | ||
| 90 | |||
| 91 | async function unblacklistVideo (videoBlacklist: MVideoBlacklist, video: MVideoFullLight) { | ||
| 92 | const videoBlacklistType = await sequelizeTypescript.transaction(async t => { | ||
| 93 | const unfederated = videoBlacklist.unfederated | ||
| 94 | const videoBlacklistType = videoBlacklist.type | ||
| 95 | |||
| 96 | await videoBlacklist.destroy({ transaction: t }) | ||
| 97 | video.VideoBlacklist = undefined | ||
| 98 | |||
| 99 | // Re federate the video | ||
| 100 | if (unfederated === true) { | ||
| 101 | await federateVideoIfNeeded(video, true, t) | ||
| 102 | } | ||
| 103 | |||
| 104 | return videoBlacklistType | ||
| 105 | }) | ||
| 106 | |||
| 107 | Notifier.Instance.notifyOnVideoUnblacklist(video) | ||
| 108 | |||
| 109 | if (videoBlacklistType === VideoBlacklistType.AUTO_BEFORE_PUBLISHED) { | ||
| 110 | Notifier.Instance.notifyOnVideoPublishedAfterRemovedFromAutoBlacklist(video) | ||
| 111 | |||
| 112 | // Delete on object so new video notifications will send | ||
| 113 | delete video.VideoBlacklist | ||
| 114 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
| 115 | } | ||
| 116 | } | ||
| 117 | |||
| 118 | // --------------------------------------------------------------------------- | ||
| 119 | |||
| 120 | export { | ||
| 121 | autoBlacklistVideoIfNeeded, | ||
| 122 | blacklistVideo, | ||
| 123 | unblacklistVideo | ||
| 124 | } | ||
| 125 | |||
| 126 | // --------------------------------------------------------------------------- | ||
| 127 | |||
| 128 | function autoBlacklistNeeded (parameters: { | ||
| 129 | video: MVideoWithBlacklistLight | ||
| 130 | isRemote: boolean | ||
| 131 | isNew: boolean | ||
| 132 | isNewFile: boolean | ||
| 133 | user?: MUser | ||
| 134 | }) { | ||
| 135 | const { user, video, isRemote, isNew, isNewFile } = parameters | ||
| 136 | |||
| 137 | // Already blacklisted | ||
| 138 | if (video.VideoBlacklist) return false | ||
| 139 | if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED || !user) return false | ||
| 140 | if (isRemote || (isNew === false && isNewFile === false)) return false | ||
| 141 | |||
| 142 | if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST)) return false | ||
| 143 | |||
| 144 | return true | ||
| 145 | } | ||
diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts deleted file mode 100644 index 8322c9ad2..000000000 --- a/server/lib/video-channel.ts +++ /dev/null | |||
| @@ -1,50 +0,0 @@ | |||
| 1 | import * as Sequelize from 'sequelize' | ||
| 2 | import { VideoChannelCreate } from '../../shared/models' | ||
| 3 | import { VideoModel } from '../models/video/video' | ||
| 4 | import { VideoChannelModel } from '../models/video/video-channel' | ||
| 5 | import { MAccountId, MChannelId } from '../types/models' | ||
| 6 | import { getLocalVideoChannelActivityPubUrl } from './activitypub/url' | ||
| 7 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
| 8 | import { buildActorInstance } from './local-actor' | ||
| 9 | |||
| 10 | async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { | ||
| 11 | const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) | ||
| 12 | const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name) | ||
| 13 | |||
| 14 | const actorInstanceCreated = await actorInstance.save({ transaction: t }) | ||
| 15 | |||
| 16 | const videoChannelData = { | ||
| 17 | name: videoChannelInfo.displayName, | ||
| 18 | description: videoChannelInfo.description, | ||
| 19 | support: videoChannelInfo.support, | ||
| 20 | accountId: account.id, | ||
| 21 | actorId: actorInstanceCreated.id | ||
| 22 | } | ||
| 23 | |||
| 24 | const videoChannel = new VideoChannelModel(videoChannelData) | ||
| 25 | |||
| 26 | const options = { transaction: t } | ||
| 27 | const videoChannelCreated = await videoChannel.save(options) | ||
| 28 | |||
| 29 | videoChannelCreated.Actor = actorInstanceCreated | ||
| 30 | |||
| 31 | // No need to send this empty video channel to followers | ||
| 32 | return videoChannelCreated | ||
| 33 | } | ||
| 34 | |||
| 35 | async function federateAllVideosOfChannel (videoChannel: MChannelId) { | ||
| 36 | const videoIds = await VideoModel.getAllIdsFromChannel(videoChannel) | ||
| 37 | |||
| 38 | for (const videoId of videoIds) { | ||
| 39 | const video = await VideoModel.loadFull(videoId) | ||
| 40 | |||
| 41 | await federateVideoIfNeeded(video, false) | ||
| 42 | } | ||
| 43 | } | ||
| 44 | |||
| 45 | // --------------------------------------------------------------------------- | ||
| 46 | |||
| 47 | export { | ||
| 48 | createLocalVideoChannel, | ||
| 49 | federateAllVideosOfChannel | ||
| 50 | } | ||
diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts deleted file mode 100644 index 6eb865f7f..000000000 --- a/server/lib/video-comment.ts +++ /dev/null | |||
| @@ -1,116 +0,0 @@ | |||
| 1 | import express from 'express' | ||
| 2 | import { cloneDeep } from 'lodash' | ||
| 3 | import * as Sequelize from 'sequelize' | ||
| 4 | import { logger } from '@server/helpers/logger' | ||
| 5 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 6 | import { ResultList } from '../../shared/models' | ||
| 7 | import { VideoCommentThreadTree } from '../../shared/models/videos/comment/video-comment.model' | ||
| 8 | import { VideoCommentModel } from '../models/video/video-comment' | ||
| 9 | import { | ||
| 10 | MAccountDefault, | ||
| 11 | MComment, | ||
| 12 | MCommentFormattable, | ||
| 13 | MCommentOwnerVideo, | ||
| 14 | MCommentOwnerVideoReply, | ||
| 15 | MVideoFullLight | ||
| 16 | } from '../types/models' | ||
| 17 | import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send' | ||
| 18 | import { getLocalVideoCommentActivityPubUrl } from './activitypub/url' | ||
| 19 | import { Hooks } from './plugins/hooks' | ||
| 20 | |||
| 21 | async function removeComment (commentArg: MComment, req: express.Request, res: express.Response) { | ||
| 22 | let videoCommentInstanceBefore: MCommentOwnerVideo | ||
| 23 | |||
| 24 | await sequelizeTypescript.transaction(async t => { | ||
| 25 | const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(commentArg.url, t) | ||
| 26 | |||
| 27 | videoCommentInstanceBefore = cloneDeep(comment) | ||
| 28 | |||
| 29 | if (comment.isOwned() || comment.Video.isOwned()) { | ||
| 30 | await sendDeleteVideoComment(comment, t) | ||
| 31 | } | ||
| 32 | |||
| 33 | comment.markAsDeleted() | ||
| 34 | |||
| 35 | await comment.save({ transaction: t }) | ||
| 36 | |||
| 37 | logger.info('Video comment %d deleted.', comment.id) | ||
| 38 | }) | ||
| 39 | |||
| 40 | Hooks.runAction('action:api.video-comment.deleted', { comment: videoCommentInstanceBefore, req, res }) | ||
| 41 | } | ||
| 42 | |||
| 43 | async function createVideoComment (obj: { | ||
| 44 | text: string | ||
| 45 | inReplyToComment: MComment | null | ||
| 46 | video: MVideoFullLight | ||
| 47 | account: MAccountDefault | ||
| 48 | }, t: Sequelize.Transaction) { | ||
| 49 | let originCommentId: number | null = null | ||
| 50 | let inReplyToCommentId: number | null = null | ||
| 51 | |||
| 52 | if (obj.inReplyToComment && obj.inReplyToComment !== null) { | ||
| 53 | originCommentId = obj.inReplyToComment.originCommentId || obj.inReplyToComment.id | ||
| 54 | inReplyToCommentId = obj.inReplyToComment.id | ||
| 55 | } | ||
| 56 | |||
| 57 | const comment = await VideoCommentModel.create({ | ||
| 58 | text: obj.text, | ||
| 59 | originCommentId, | ||
| 60 | inReplyToCommentId, | ||
| 61 | videoId: obj.video.id, | ||
| 62 | accountId: obj.account.id, | ||
| 63 | url: new Date().toISOString() | ||
| 64 | }, { transaction: t, validate: false }) | ||
| 65 | |||
| 66 | comment.url = getLocalVideoCommentActivityPubUrl(obj.video, comment) | ||
| 67 | |||
| 68 | const savedComment: MCommentOwnerVideoReply = await comment.save({ transaction: t }) | ||
| 69 | savedComment.InReplyToVideoComment = obj.inReplyToComment | ||
| 70 | savedComment.Video = obj.video | ||
| 71 | savedComment.Account = obj.account | ||
| 72 | |||
| 73 | await sendCreateVideoComment(savedComment, t) | ||
| 74 | |||
| 75 | return savedComment | ||
| 76 | } | ||
| 77 | |||
| 78 | function buildFormattedCommentTree (resultList: ResultList<MCommentFormattable>): VideoCommentThreadTree { | ||
| 79 | // Comments are sorted by id ASC | ||
| 80 | const comments = resultList.data | ||
| 81 | |||
| 82 | const comment = comments.shift() | ||
| 83 | const thread: VideoCommentThreadTree = { | ||
| 84 | comment: comment.toFormattedJSON(), | ||
| 85 | children: [] | ||
| 86 | } | ||
| 87 | const idx = { | ||
| 88 | [comment.id]: thread | ||
| 89 | } | ||
| 90 | |||
| 91 | while (comments.length !== 0) { | ||
| 92 | const childComment = comments.shift() | ||
| 93 | |||
| 94 | const childCommentThread: VideoCommentThreadTree = { | ||
| 95 | comment: childComment.toFormattedJSON(), | ||
| 96 | children: [] | ||
| 97 | } | ||
| 98 | |||
| 99 | const parentCommentThread = idx[childComment.inReplyToCommentId] | ||
| 100 | // Maybe the parent comment was blocked by the admin/user | ||
| 101 | if (!parentCommentThread) continue | ||
| 102 | |||
| 103 | parentCommentThread.children.push(childCommentThread) | ||
| 104 | idx[childComment.id] = childCommentThread | ||
| 105 | } | ||
| 106 | |||
| 107 | return thread | ||
| 108 | } | ||
| 109 | |||
| 110 | // --------------------------------------------------------------------------- | ||
| 111 | |||
| 112 | export { | ||
| 113 | removeComment, | ||
| 114 | createVideoComment, | ||
| 115 | buildFormattedCommentTree | ||
| 116 | } | ||
diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts deleted file mode 100644 index 46af67ccd..000000000 --- a/server/lib/video-file.ts +++ /dev/null | |||
| @@ -1,145 +0,0 @@ | |||
| 1 | import { FfprobeData } from 'fluent-ffmpeg' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { VideoFileModel } from '@server/models/video/video-file' | ||
| 4 | import { MVideoWithAllFiles } from '@server/types/models' | ||
| 5 | import { getLowercaseExtension } from '@shared/core-utils' | ||
| 6 | import { getFileSize } from '@shared/extra-utils' | ||
| 7 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' | ||
| 8 | import { VideoFileMetadata, VideoResolution } from '@shared/models' | ||
| 9 | import { lTags } from './object-storage/shared' | ||
| 10 | import { generateHLSVideoFilename, generateWebVideoFilename } from './paths' | ||
| 11 | import { VideoPathManager } from './video-path-manager' | ||
| 12 | |||
| 13 | async function buildNewFile (options: { | ||
| 14 | path: string | ||
| 15 | mode: 'web-video' | 'hls' | ||
| 16 | }) { | ||
| 17 | const { path, mode } = options | ||
| 18 | |||
| 19 | const probe = await ffprobePromise(path) | ||
| 20 | const size = await getFileSize(path) | ||
| 21 | |||
| 22 | const videoFile = new VideoFileModel({ | ||
| 23 | extname: getLowercaseExtension(path), | ||
| 24 | size, | ||
| 25 | metadata: await buildFileMetadata(path, probe) | ||
| 26 | }) | ||
| 27 | |||
| 28 | if (await isAudioFile(path, probe)) { | ||
| 29 | videoFile.resolution = VideoResolution.H_NOVIDEO | ||
| 30 | } else { | ||
| 31 | videoFile.fps = await getVideoStreamFPS(path, probe) | ||
| 32 | videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution | ||
| 33 | } | ||
| 34 | |||
| 35 | videoFile.filename = mode === 'web-video' | ||
| 36 | ? generateWebVideoFilename(videoFile.resolution, videoFile.extname) | ||
| 37 | : generateHLSVideoFilename(videoFile.resolution) | ||
| 38 | |||
| 39 | return videoFile | ||
| 40 | } | ||
| 41 | |||
| 42 | // --------------------------------------------------------------------------- | ||
| 43 | |||
| 44 | async function removeHLSPlaylist (video: MVideoWithAllFiles) { | ||
| 45 | const hls = video.getHLSPlaylist() | ||
| 46 | if (!hls) return | ||
| 47 | |||
| 48 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 49 | |||
| 50 | try { | ||
| 51 | await video.removeStreamingPlaylistFiles(hls) | ||
| 52 | await hls.destroy() | ||
| 53 | |||
| 54 | video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id) | ||
| 55 | } finally { | ||
| 56 | videoFileMutexReleaser() | ||
| 57 | } | ||
| 58 | } | ||
| 59 | |||
| 60 | async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { | ||
| 61 | logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid)) | ||
| 62 | |||
| 63 | const hls = video.getHLSPlaylist() | ||
| 64 | const files = hls.VideoFiles | ||
| 65 | |||
| 66 | if (files.length === 1) { | ||
| 67 | await removeHLSPlaylist(video) | ||
| 68 | return undefined | ||
| 69 | } | ||
| 70 | |||
| 71 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 72 | |||
| 73 | try { | ||
| 74 | const toDelete = files.find(f => f.id === fileToDeleteId) | ||
| 75 | await video.removeStreamingPlaylistVideoFile(video.getHLSPlaylist(), toDelete) | ||
| 76 | await toDelete.destroy() | ||
| 77 | |||
| 78 | hls.VideoFiles = hls.VideoFiles.filter(f => f.id !== toDelete.id) | ||
| 79 | } finally { | ||
| 80 | videoFileMutexReleaser() | ||
| 81 | } | ||
| 82 | |||
| 83 | return hls | ||
| 84 | } | ||
| 85 | |||
| 86 | // --------------------------------------------------------------------------- | ||
| 87 | |||
| 88 | async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { | ||
| 89 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 90 | |||
| 91 | try { | ||
| 92 | for (const file of video.VideoFiles) { | ||
| 93 | await video.removeWebVideoFile(file) | ||
| 94 | await file.destroy() | ||
| 95 | } | ||
| 96 | |||
| 97 | video.VideoFiles = [] | ||
| 98 | } finally { | ||
| 99 | videoFileMutexReleaser() | ||
| 100 | } | ||
| 101 | |||
| 102 | return video | ||
| 103 | } | ||
| 104 | |||
| 105 | async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) { | ||
| 106 | const files = video.VideoFiles | ||
| 107 | |||
| 108 | if (files.length === 1) { | ||
| 109 | return removeAllWebVideoFiles(video) | ||
| 110 | } | ||
| 111 | |||
| 112 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
| 113 | try { | ||
| 114 | const toDelete = files.find(f => f.id === fileToDeleteId) | ||
| 115 | await video.removeWebVideoFile(toDelete) | ||
| 116 | await toDelete.destroy() | ||
| 117 | |||
| 118 | video.VideoFiles = files.filter(f => f.id !== toDelete.id) | ||
| 119 | } finally { | ||
| 120 | videoFileMutexReleaser() | ||
| 121 | } | ||
| 122 | |||
| 123 | return video | ||
| 124 | } | ||
| 125 | |||
| 126 | // --------------------------------------------------------------------------- | ||
| 127 | |||
| 128 | async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { | ||
| 129 | const metadata = existingProbe || await ffprobePromise(path) | ||
| 130 | |||
| 131 | return new VideoFileMetadata(metadata) | ||
| 132 | } | ||
| 133 | |||
| 134 | // --------------------------------------------------------------------------- | ||
| 135 | |||
| 136 | export { | ||
| 137 | buildNewFile, | ||
| 138 | |||
| 139 | removeHLSPlaylist, | ||
| 140 | removeHLSFile, | ||
| 141 | removeAllWebVideoFiles, | ||
| 142 | removeWebVideoFile, | ||
| 143 | |||
| 144 | buildFileMetadata | ||
| 145 | } | ||
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts deleted file mode 100644 index 133544bb2..000000000 --- a/server/lib/video-path-manager.ts +++ /dev/null | |||
| @@ -1,174 +0,0 @@ | |||
| 1 | import { Mutex } from 'async-mutex' | ||
| 2 | import { remove } from 'fs-extra' | ||
| 3 | import { extname, join } from 'path' | ||
| 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 5 | import { extractVideo } from '@server/helpers/video' | ||
| 6 | import { CONFIG } from '@server/initializers/config' | ||
| 7 | import { DIRECTORIES } from '@server/initializers/constants' | ||
| 8 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models' | ||
| 9 | import { buildUUID } from '@shared/extra-utils' | ||
| 10 | import { VideoStorage } from '@shared/models' | ||
| 11 | import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage' | ||
| 12 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' | ||
| 13 | import { isVideoInPrivateDirectory } from './video-privacy' | ||
| 14 | |||
| 15 | type MakeAvailableCB <T> = (path: string) => Promise<T> | T | ||
| 16 | |||
| 17 | const lTags = loggerTagsFactory('video-path-manager') | ||
| 18 | |||
| 19 | class VideoPathManager { | ||
| 20 | |||
| 21 | private static instance: VideoPathManager | ||
| 22 | |||
| 23 | // Key is a video UUID | ||
| 24 | private readonly videoFileMutexStore = new Map<string, Mutex>() | ||
| 25 | |||
| 26 | private constructor () {} | ||
| 27 | |||
| 28 | getFSHLSOutputPath (video: MVideo, filename?: string) { | ||
| 29 | const base = getHLSDirectory(video) | ||
| 30 | if (!filename) return base | ||
| 31 | |||
| 32 | return join(base, filename) | ||
| 33 | } | ||
| 34 | |||
| 35 | getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
| 36 | if (videoFile.isHLS()) { | ||
| 37 | const video = extractVideo(videoOrPlaylist) | ||
| 38 | |||
| 39 | return join(getHLSRedundancyDirectory(video), videoFile.filename) | ||
| 40 | } | ||
| 41 | |||
| 42 | return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) | ||
| 43 | } | ||
| 44 | |||
| 45 | getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
| 46 | const video = extractVideo(videoOrPlaylist) | ||
| 47 | |||
| 48 | if (videoFile.isHLS()) { | ||
| 49 | return join(getHLSDirectory(video), videoFile.filename) | ||
| 50 | } | ||
| 51 | |||
| 52 | if (isVideoInPrivateDirectory(video.privacy)) { | ||
| 53 | return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename) | ||
| 54 | } | ||
| 55 | |||
| 56 | return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename) | ||
| 57 | } | ||
| 58 | |||
| 59 | async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) { | ||
| 60 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
| 61 | return this.makeAvailableFactory( | ||
| 62 | () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile), | ||
| 63 | false, | ||
| 64 | cb | ||
| 65 | ) | ||
| 66 | } | ||
| 67 | |||
| 68 | const destination = this.buildTMPDestination(videoFile.filename) | ||
| 69 | |||
| 70 | if (videoFile.isHLS()) { | ||
| 71 | const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist | ||
| 72 | |||
| 73 | return this.makeAvailableFactory( | ||
| 74 | () => makeHLSFileAvailable(playlist, videoFile.filename, destination), | ||
| 75 | true, | ||
| 76 | cb | ||
| 77 | ) | ||
| 78 | } | ||
| 79 | |||
| 80 | return this.makeAvailableFactory( | ||
| 81 | () => makeWebVideoFileAvailable(videoFile.filename, destination), | ||
| 82 | true, | ||
| 83 | cb | ||
| 84 | ) | ||
| 85 | } | ||
| 86 | |||
| 87 | async makeAvailableResolutionPlaylistFile <T> (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) { | ||
| 88 | const filename = getHlsResolutionPlaylistFilename(videoFile.filename) | ||
| 89 | |||
| 90 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
| 91 | return this.makeAvailableFactory( | ||
| 92 | () => join(getHLSDirectory(videoFile.getVideo()), filename), | ||
| 93 | false, | ||
| 94 | cb | ||
| 95 | ) | ||
| 96 | } | ||
| 97 | |||
| 98 | const playlist = videoFile.VideoStreamingPlaylist | ||
| 99 | return this.makeAvailableFactory( | ||
| 100 | () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), | ||
| 101 | true, | ||
| 102 | cb | ||
| 103 | ) | ||
| 104 | } | ||
| 105 | |||
| 106 | async makeAvailablePlaylistFile <T> (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB<T>) { | ||
| 107 | if (playlist.storage === VideoStorage.FILE_SYSTEM) { | ||
| 108 | return this.makeAvailableFactory( | ||
| 109 | () => join(getHLSDirectory(playlist.Video), filename), | ||
| 110 | false, | ||
| 111 | cb | ||
| 112 | ) | ||
| 113 | } | ||
| 114 | |||
| 115 | return this.makeAvailableFactory( | ||
| 116 | () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), | ||
| 117 | true, | ||
| 118 | cb | ||
| 119 | ) | ||
| 120 | } | ||
| 121 | |||
| 122 | async lockFiles (videoUUID: string) { | ||
| 123 | if (!this.videoFileMutexStore.has(videoUUID)) { | ||
| 124 | this.videoFileMutexStore.set(videoUUID, new Mutex()) | ||
| 125 | } | ||
| 126 | |||
| 127 | const mutex = this.videoFileMutexStore.get(videoUUID) | ||
| 128 | const releaser = await mutex.acquire() | ||
| 129 | |||
| 130 | logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID)) | ||
| 131 | |||
| 132 | return releaser | ||
| 133 | } | ||
| 134 | |||
| 135 | unlockFiles (videoUUID: string) { | ||
| 136 | const mutex = this.videoFileMutexStore.get(videoUUID) | ||
| 137 | |||
| 138 | mutex.release() | ||
| 139 | |||
| 140 | logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID)) | ||
| 141 | } | ||
| 142 | |||
| 143 | private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { | ||
| 144 | let result: T | ||
| 145 | |||
| 146 | const destination = await method() | ||
| 147 | |||
| 148 | try { | ||
| 149 | result = await cb(destination) | ||
| 150 | } catch (err) { | ||
| 151 | if (destination && clean) await remove(destination) | ||
| 152 | throw err | ||
| 153 | } | ||
| 154 | |||
| 155 | if (clean) await remove(destination) | ||
| 156 | |||
| 157 | return result | ||
| 158 | } | ||
| 159 | |||
| 160 | private buildTMPDestination (filename: string) { | ||
| 161 | return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename)) | ||
| 162 | |||
| 163 | } | ||
| 164 | |||
| 165 | static get Instance () { | ||
| 166 | return this.instance || (this.instance = new this()) | ||
| 167 | } | ||
| 168 | } | ||
| 169 | |||
| 170 | // --------------------------------------------------------------------------- | ||
| 171 | |||
| 172 | export { | ||
| 173 | VideoPathManager | ||
| 174 | } | ||
diff --git a/server/lib/video-playlist.ts b/server/lib/video-playlist.ts deleted file mode 100644 index a1af2e1af..000000000 --- a/server/lib/video-playlist.ts +++ /dev/null | |||
| @@ -1,30 +0,0 @@ | |||
| 1 | import * as Sequelize from 'sequelize' | ||
| 2 | import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model' | ||
| 3 | import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model' | ||
| 4 | import { VideoPlaylistModel } from '../models/video/video-playlist' | ||
| 5 | import { MAccount } from '../types/models' | ||
| 6 | import { MVideoPlaylistOwner } from '../types/models/video/video-playlist' | ||
| 7 | import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url' | ||
| 8 | |||
| 9 | async function createWatchLaterPlaylist (account: MAccount, t: Sequelize.Transaction) { | ||
| 10 | const videoPlaylist: MVideoPlaylistOwner = new VideoPlaylistModel({ | ||
| 11 | name: 'Watch later', | ||
| 12 | privacy: VideoPlaylistPrivacy.PRIVATE, | ||
| 13 | type: VideoPlaylistType.WATCH_LATER, | ||
| 14 | ownerAccountId: account.id | ||
| 15 | }) | ||
| 16 | |||
| 17 | videoPlaylist.url = getLocalVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object | ||
| 18 | |||
| 19 | await videoPlaylist.save({ transaction: t }) | ||
| 20 | |||
| 21 | videoPlaylist.OwnerAccount = account | ||
| 22 | |||
| 23 | return videoPlaylist | ||
| 24 | } | ||
| 25 | |||
| 26 | // --------------------------------------------------------------------------- | ||
| 27 | |||
| 28 | export { | ||
| 29 | createWatchLaterPlaylist | ||
| 30 | } | ||
diff --git a/server/lib/video-pre-import.ts b/server/lib/video-pre-import.ts deleted file mode 100644 index fcb9f77d7..000000000 --- a/server/lib/video-pre-import.ts +++ /dev/null | |||
| @@ -1,323 +0,0 @@ | |||
| 1 | import { remove } from 'fs-extra' | ||
| 2 | import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils' | ||
| 3 | import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' | ||
| 4 | import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' | ||
| 5 | import { isResolvingToUnicastOnly } from '@server/helpers/dns' | ||
| 6 | import { logger } from '@server/helpers/logger' | ||
| 7 | import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl' | ||
| 8 | import { CONFIG } from '@server/initializers/config' | ||
| 9 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 10 | import { Hooks } from '@server/lib/plugins/hooks' | ||
| 11 | import { ServerConfigManager } from '@server/lib/server-config-manager' | ||
| 12 | import { setVideoTags } from '@server/lib/video' | ||
| 13 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | ||
| 14 | import { VideoModel } from '@server/models/video/video' | ||
| 15 | import { VideoCaptionModel } from '@server/models/video/video-caption' | ||
| 16 | import { VideoImportModel } from '@server/models/video/video-import' | ||
| 17 | import { FilteredModelAttributes } from '@server/types' | ||
| 18 | import { | ||
| 19 | MChannelAccountDefault, | ||
| 20 | MChannelSync, | ||
| 21 | MThumbnail, | ||
| 22 | MUser, | ||
| 23 | MVideoAccountDefault, | ||
| 24 | MVideoCaption, | ||
| 25 | MVideoImportFormattable, | ||
| 26 | MVideoTag, | ||
| 27 | MVideoThumbnail, | ||
| 28 | MVideoWithBlacklistLight | ||
| 29 | } from '@server/types/models' | ||
| 30 | import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' | ||
| 31 | import { getLocalVideoActivityPubUrl } from './activitypub/url' | ||
| 32 | import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail' | ||
| 33 | import { VideoPasswordModel } from '@server/models/video/video-password' | ||
| 34 | |||
| 35 | class YoutubeDlImportError extends Error { | ||
| 36 | code: YoutubeDlImportError.CODE | ||
| 37 | cause?: Error // Property to remove once ES2022 is used | ||
| 38 | constructor ({ message, code }) { | ||
| 39 | super(message) | ||
| 40 | this.code = code | ||
| 41 | } | ||
| 42 | |||
| 43 | static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) { | ||
| 44 | const ytDlErr = new this({ message: message ?? err.message, code }) | ||
| 45 | ytDlErr.cause = err | ||
| 46 | ytDlErr.stack = err.stack // Useless once ES2022 is used | ||
| 47 | return ytDlErr | ||
| 48 | } | ||
| 49 | } | ||
| 50 | |||
| 51 | namespace YoutubeDlImportError { | ||
| 52 | export enum CODE { | ||
| 53 | FETCH_ERROR, | ||
| 54 | NOT_ONLY_UNICAST_URL | ||
| 55 | } | ||
| 56 | } | ||
| 57 | |||
| 58 | // --------------------------------------------------------------------------- | ||
| 59 | |||
| 60 | async function insertFromImportIntoDB (parameters: { | ||
| 61 | video: MVideoThumbnail | ||
| 62 | thumbnailModel: MThumbnail | ||
| 63 | previewModel: MThumbnail | ||
| 64 | videoChannel: MChannelAccountDefault | ||
| 65 | tags: string[] | ||
| 66 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> | ||
| 67 | user: MUser | ||
| 68 | videoPasswords?: string[] | ||
| 69 | }): Promise<MVideoImportFormattable> { | ||
| 70 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user, videoPasswords } = parameters | ||
| 71 | |||
| 72 | const videoImport = await sequelizeTypescript.transaction(async t => { | ||
| 73 | const sequelizeOptions = { transaction: t } | ||
| 74 | |||
| 75 | // Save video object in database | ||
| 76 | const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) | ||
| 77 | videoCreated.VideoChannel = videoChannel | ||
| 78 | |||
| 79 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | ||
| 80 | if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) | ||
| 81 | |||
| 82 | if (videoCreated.privacy === VideoPrivacy.PASSWORD_PROTECTED) { | ||
| 83 | await VideoPasswordModel.addPasswords(videoPasswords, video.id, t) | ||
| 84 | } | ||
| 85 | |||
| 86 | await autoBlacklistVideoIfNeeded({ | ||
| 87 | video: videoCreated, | ||
| 88 | user, | ||
| 89 | notify: false, | ||
| 90 | isRemote: false, | ||
| 91 | isNew: true, | ||
| 92 | isNewFile: true, | ||
| 93 | transaction: t | ||
| 94 | }) | ||
| 95 | |||
| 96 | await setVideoTags({ video: videoCreated, tags, transaction: t }) | ||
| 97 | |||
| 98 | // Create video import object in database | ||
| 99 | const videoImport = await VideoImportModel.create( | ||
| 100 | Object.assign({ videoId: videoCreated.id }, videoImportAttributes), | ||
| 101 | sequelizeOptions | ||
| 102 | ) as MVideoImportFormattable | ||
| 103 | videoImport.Video = videoCreated | ||
| 104 | |||
| 105 | return videoImport | ||
| 106 | }) | ||
| 107 | |||
| 108 | return videoImport | ||
| 109 | } | ||
| 110 | |||
| 111 | async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: { | ||
| 112 | channelId: number | ||
| 113 | importData: YoutubeDLInfo | ||
| 114 | importDataOverride?: Partial<VideoImportCreate> | ||
| 115 | importType: 'url' | 'torrent' | ||
| 116 | }): Promise<MVideoThumbnail> { | ||
| 117 | let videoData = { | ||
| 118 | name: importDataOverride?.name || importData.name || 'Unknown name', | ||
| 119 | remote: false, | ||
| 120 | category: importDataOverride?.category || importData.category, | ||
| 121 | licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, | ||
| 122 | language: importDataOverride?.language || importData.language, | ||
| 123 | commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, | ||
| 124 | downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, | ||
| 125 | waitTranscoding: importDataOverride?.waitTranscoding ?? true, | ||
| 126 | state: VideoState.TO_IMPORT, | ||
| 127 | nsfw: importDataOverride?.nsfw || importData.nsfw || false, | ||
| 128 | description: importDataOverride?.description || importData.description, | ||
| 129 | support: importDataOverride?.support || null, | ||
| 130 | privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE, | ||
| 131 | duration: 0, // duration will be set by the import job | ||
| 132 | channelId, | ||
| 133 | originallyPublishedAt: importDataOverride?.originallyPublishedAt | ||
| 134 | ? new Date(importDataOverride?.originallyPublishedAt) | ||
| 135 | : importData.originallyPublishedAtWithoutTime | ||
| 136 | } | ||
| 137 | |||
| 138 | videoData = await Hooks.wrapObject( | ||
| 139 | videoData, | ||
| 140 | importType === 'url' | ||
| 141 | ? 'filter:api.video.import-url.video-attribute.result' | ||
| 142 | : 'filter:api.video.import-torrent.video-attribute.result' | ||
| 143 | ) | ||
| 144 | |||
| 145 | const video = new VideoModel(videoData) | ||
| 146 | video.url = getLocalVideoActivityPubUrl(video) | ||
| 147 | |||
| 148 | return video | ||
| 149 | } | ||
| 150 | |||
| 151 | async function buildYoutubeDLImport (options: { | ||
| 152 | targetUrl: string | ||
| 153 | channel: MChannelAccountDefault | ||
| 154 | user: MUser | ||
| 155 | channelSync?: MChannelSync | ||
| 156 | importDataOverride?: Partial<VideoImportCreate> | ||
| 157 | thumbnailFilePath?: string | ||
| 158 | previewFilePath?: string | ||
| 159 | }) { | ||
| 160 | const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options | ||
| 161 | |||
| 162 | const youtubeDL = new YoutubeDLWrapper( | ||
| 163 | targetUrl, | ||
| 164 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | ||
| 165 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | ||
| 166 | ) | ||
| 167 | |||
| 168 | // Get video infos | ||
| 169 | let youtubeDLInfo: YoutubeDLInfo | ||
| 170 | try { | ||
| 171 | youtubeDLInfo = await youtubeDL.getInfoForDownload() | ||
| 172 | } catch (err) { | ||
| 173 | throw YoutubeDlImportError.fromError( | ||
| 174 | err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}` | ||
| 175 | ) | ||
| 176 | } | ||
| 177 | |||
| 178 | if (!await hasUnicastURLsOnly(youtubeDLInfo)) { | ||
| 179 | throw new YoutubeDlImportError({ | ||
| 180 | message: 'Cannot use non unicast IP as targetUrl.', | ||
| 181 | code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL | ||
| 182 | }) | ||
| 183 | } | ||
| 184 | |||
| 185 | const video = await buildVideoFromImport({ | ||
| 186 | channelId: channel.id, | ||
| 187 | importData: youtubeDLInfo, | ||
| 188 | importDataOverride, | ||
| 189 | importType: 'url' | ||
| 190 | }) | ||
| 191 | |||
| 192 | const thumbnailModel = await forgeThumbnail({ | ||
| 193 | inputPath: thumbnailFilePath, | ||
| 194 | downloadUrl: youtubeDLInfo.thumbnailUrl, | ||
| 195 | video, | ||
| 196 | type: ThumbnailType.MINIATURE | ||
| 197 | }) | ||
| 198 | |||
| 199 | const previewModel = await forgeThumbnail({ | ||
| 200 | inputPath: previewFilePath, | ||
| 201 | downloadUrl: youtubeDLInfo.thumbnailUrl, | ||
| 202 | video, | ||
| 203 | type: ThumbnailType.PREVIEW | ||
| 204 | }) | ||
| 205 | |||
| 206 | const videoImport = await insertFromImportIntoDB({ | ||
| 207 | video, | ||
| 208 | thumbnailModel, | ||
| 209 | previewModel, | ||
| 210 | videoChannel: channel, | ||
| 211 | tags: importDataOverride?.tags || youtubeDLInfo.tags, | ||
| 212 | user, | ||
| 213 | videoImportAttributes: { | ||
| 214 | targetUrl, | ||
| 215 | state: VideoImportState.PENDING, | ||
| 216 | userId: user.id, | ||
| 217 | videoChannelSyncId: channelSync?.id | ||
| 218 | }, | ||
| 219 | videoPasswords: importDataOverride.videoPasswords | ||
| 220 | }) | ||
| 221 | |||
| 222 | // Get video subtitles | ||
| 223 | await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) | ||
| 224 | |||
| 225 | let fileExt = `.${youtubeDLInfo.ext}` | ||
| 226 | if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' | ||
| 227 | |||
| 228 | const payload: VideoImportPayload = { | ||
| 229 | type: 'youtube-dl' as 'youtube-dl', | ||
| 230 | videoImportId: videoImport.id, | ||
| 231 | fileExt, | ||
| 232 | // If part of a sync process, there is a parent job that will aggregate children results | ||
| 233 | preventException: !!channelSync | ||
| 234 | } | ||
| 235 | |||
| 236 | return { | ||
| 237 | videoImport, | ||
| 238 | job: { type: 'video-import' as 'video-import', payload } | ||
| 239 | } | ||
| 240 | } | ||
| 241 | |||
| 242 | // --------------------------------------------------------------------------- | ||
| 243 | |||
| 244 | export { | ||
| 245 | buildYoutubeDLImport, | ||
| 246 | YoutubeDlImportError, | ||
| 247 | insertFromImportIntoDB, | ||
| 248 | buildVideoFromImport | ||
| 249 | } | ||
| 250 | |||
| 251 | // --------------------------------------------------------------------------- | ||
| 252 | |||
| 253 | async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { | ||
| 254 | inputPath?: string | ||
| 255 | downloadUrl?: string | ||
| 256 | video: MVideoThumbnail | ||
| 257 | type: ThumbnailType | ||
| 258 | }): Promise<MThumbnail> { | ||
| 259 | if (inputPath) { | ||
| 260 | return updateLocalVideoMiniatureFromExisting({ | ||
| 261 | inputPath, | ||
| 262 | video, | ||
| 263 | type, | ||
| 264 | automaticallyGenerated: false | ||
| 265 | }) | ||
| 266 | } | ||
| 267 | |||
| 268 | if (downloadUrl) { | ||
| 269 | try { | ||
| 270 | return await updateLocalVideoMiniatureFromUrl({ downloadUrl, video, type }) | ||
| 271 | } catch (err) { | ||
| 272 | logger.warn('Cannot process thumbnail %s from youtube-dl.', downloadUrl, { err }) | ||
| 273 | } | ||
| 274 | } | ||
| 275 | |||
| 276 | return null | ||
| 277 | } | ||
| 278 | |||
| 279 | async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { | ||
| 280 | try { | ||
| 281 | const subtitles = await youtubeDL.getSubtitles() | ||
| 282 | |||
| 283 | logger.info('Found %s subtitles candidates from youtube-dl import %s.', subtitles.length, targetUrl) | ||
| 284 | |||
| 285 | for (const subtitle of subtitles) { | ||
| 286 | if (!await isVTTFileValid(subtitle.path)) { | ||
| 287 | logger.info('%s is not a valid youtube-dl subtitle, skipping', subtitle.path) | ||
| 288 | await remove(subtitle.path) | ||
| 289 | continue | ||
| 290 | } | ||
| 291 | |||
| 292 | const videoCaption = new VideoCaptionModel({ | ||
| 293 | videoId, | ||
| 294 | language: subtitle.language, | ||
| 295 | filename: VideoCaptionModel.generateCaptionName(subtitle.language) | ||
| 296 | }) as MVideoCaption | ||
| 297 | |||
| 298 | // Move physical file | ||
| 299 | await moveAndProcessCaptionFile(subtitle, videoCaption) | ||
| 300 | |||
| 301 | await sequelizeTypescript.transaction(async t => { | ||
| 302 | await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) | ||
| 303 | }) | ||
| 304 | |||
| 305 | logger.info('Added %s youtube-dl subtitle', subtitle.path) | ||
| 306 | } | ||
| 307 | } catch (err) { | ||
| 308 | logger.warn('Cannot get video subtitles.', { err }) | ||
| 309 | } | ||
| 310 | } | ||
| 311 | |||
| 312 | async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { | ||
| 313 | const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) | ||
| 314 | const uniqHosts = new Set(hosts) | ||
| 315 | |||
| 316 | for (const h of uniqHosts) { | ||
| 317 | if (await isResolvingToUnicastOnly(h) !== true) { | ||
| 318 | return false | ||
| 319 | } | ||
| 320 | } | ||
| 321 | |||
| 322 | return true | ||
| 323 | } | ||
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts deleted file mode 100644 index 5dd4d9781..000000000 --- a/server/lib/video-privacy.ts +++ /dev/null | |||
| @@ -1,133 +0,0 @@ | |||
| 1 | import { move } from 'fs-extra' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { DIRECTORIES } from '@server/initializers/constants' | ||
| 5 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
| 6 | import { VideoPrivacy, VideoStorage } from '@shared/models' | ||
| 7 | import { updateHLSFilesACL, updateWebVideoFileACL } from './object-storage' | ||
| 8 | |||
| 9 | const validPrivacySet = new Set([ | ||
| 10 | VideoPrivacy.PRIVATE, | ||
| 11 | VideoPrivacy.INTERNAL, | ||
| 12 | VideoPrivacy.PASSWORD_PROTECTED | ||
| 13 | ]) | ||
| 14 | |||
| 15 | function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { | ||
| 16 | if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { | ||
| 17 | video.publishedAt = new Date() | ||
| 18 | } | ||
| 19 | |||
| 20 | video.privacy = newPrivacy | ||
| 21 | } | ||
| 22 | |||
| 23 | function isVideoInPrivateDirectory (privacy) { | ||
| 24 | return validPrivacySet.has(privacy) | ||
| 25 | } | ||
| 26 | |||
| 27 | function isVideoInPublicDirectory (privacy: VideoPrivacy) { | ||
| 28 | return !isVideoInPrivateDirectory(privacy) | ||
| 29 | } | ||
| 30 | |||
| 31 | async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) { | ||
| 32 | // Now public, previously private | ||
| 33 | if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) { | ||
| 34 | await moveFiles({ type: 'private-to-public', video }) | ||
| 35 | |||
| 36 | return true | ||
| 37 | } | ||
| 38 | |||
| 39 | // Now private, previously public | ||
| 40 | if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) { | ||
| 41 | await moveFiles({ type: 'public-to-private', video }) | ||
| 42 | |||
| 43 | return true | ||
| 44 | } | ||
| 45 | |||
| 46 | return false | ||
| 47 | } | ||
| 48 | |||
| 49 | export { | ||
| 50 | setVideoPrivacy, | ||
| 51 | |||
| 52 | isVideoInPrivateDirectory, | ||
| 53 | isVideoInPublicDirectory, | ||
| 54 | |||
| 55 | moveFilesIfPrivacyChanged | ||
| 56 | } | ||
| 57 | |||
| 58 | // --------------------------------------------------------------------------- | ||
| 59 | |||
| 60 | type MoveType = 'private-to-public' | 'public-to-private' | ||
| 61 | |||
| 62 | async function moveFiles (options: { | ||
| 63 | type: MoveType | ||
| 64 | video: MVideoFullLight | ||
| 65 | }) { | ||
| 66 | const { type, video } = options | ||
| 67 | |||
| 68 | for (const file of video.VideoFiles) { | ||
| 69 | if (file.storage === VideoStorage.FILE_SYSTEM) { | ||
| 70 | await moveWebVideoFileOnFS(type, video, file) | ||
| 71 | } else { | ||
| 72 | await updateWebVideoFileACL(video, file) | ||
| 73 | } | ||
| 74 | } | ||
| 75 | |||
| 76 | const hls = video.getHLSPlaylist() | ||
| 77 | |||
| 78 | if (hls) { | ||
| 79 | if (hls.storage === VideoStorage.FILE_SYSTEM) { | ||
| 80 | await moveHLSFilesOnFS(type, video) | ||
| 81 | } else { | ||
| 82 | await updateHLSFilesACL(hls) | ||
| 83 | } | ||
| 84 | } | ||
| 85 | } | ||
| 86 | |||
| 87 | async function moveWebVideoFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { | ||
| 88 | const directories = getWebVideoDirectories(type) | ||
| 89 | |||
| 90 | const source = join(directories.old, file.filename) | ||
| 91 | const destination = join(directories.new, file.filename) | ||
| 92 | |||
| 93 | try { | ||
| 94 | logger.info('Moving web video files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | ||
| 95 | |||
| 96 | await move(source, destination) | ||
| 97 | } catch (err) { | ||
| 98 | logger.error('Cannot move web video file %s to %s after privacy change', source, destination, { err }) | ||
| 99 | } | ||
| 100 | } | ||
| 101 | |||
| 102 | function getWebVideoDirectories (moveType: MoveType) { | ||
| 103 | if (moveType === 'private-to-public') { | ||
| 104 | return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } | ||
| 105 | } | ||
| 106 | |||
| 107 | return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } | ||
| 108 | } | ||
| 109 | |||
| 110 | // --------------------------------------------------------------------------- | ||
| 111 | |||
| 112 | async function moveHLSFilesOnFS (type: MoveType, video: MVideo) { | ||
| 113 | const directories = getHLSDirectories(type) | ||
| 114 | |||
| 115 | const source = join(directories.old, video.uuid) | ||
| 116 | const destination = join(directories.new, video.uuid) | ||
| 117 | |||
| 118 | try { | ||
| 119 | logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | ||
| 120 | |||
| 121 | await move(source, destination) | ||
| 122 | } catch (err) { | ||
| 123 | logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) | ||
| 124 | } | ||
| 125 | } | ||
| 126 | |||
| 127 | function getHLSDirectories (moveType: MoveType) { | ||
| 128 | if (moveType === 'private-to-public') { | ||
| 129 | return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } | ||
| 130 | } | ||
| 131 | |||
| 132 | return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } | ||
| 133 | } | ||
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts deleted file mode 100644 index 893725d85..000000000 --- a/server/lib/video-state.ts +++ /dev/null | |||
| @@ -1,154 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize' | ||
| 2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 3 | import { logger } from '@server/helpers/logger' | ||
| 4 | import { CONFIG } from '@server/initializers/config' | ||
| 5 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 6 | import { VideoModel } from '@server/models/video/video' | ||
| 7 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 8 | import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models' | ||
| 9 | import { VideoState } from '@shared/models' | ||
| 10 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
| 11 | import { JobQueue } from './job-queue' | ||
| 12 | import { Notifier } from './notifier' | ||
| 13 | import { buildMoveToObjectStorageJob } from './video' | ||
| 14 | |||
| 15 | function buildNextVideoState (currentState?: VideoState) { | ||
| 16 | if (currentState === VideoState.PUBLISHED) { | ||
| 17 | throw new Error('Video is already in its final state') | ||
| 18 | } | ||
| 19 | |||
| 20 | if ( | ||
| 21 | currentState !== VideoState.TO_EDIT && | ||
| 22 | currentState !== VideoState.TO_TRANSCODE && | ||
| 23 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
| 24 | CONFIG.TRANSCODING.ENABLED | ||
| 25 | ) { | ||
| 26 | return VideoState.TO_TRANSCODE | ||
| 27 | } | ||
| 28 | |||
| 29 | if ( | ||
| 30 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
| 31 | CONFIG.OBJECT_STORAGE.ENABLED | ||
| 32 | ) { | ||
| 33 | return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE | ||
| 34 | } | ||
| 35 | |||
| 36 | return VideoState.PUBLISHED | ||
| 37 | } | ||
| 38 | |||
| 39 | function moveToNextState (options: { | ||
| 40 | video: MVideoUUID | ||
| 41 | previousVideoState?: VideoState | ||
| 42 | isNewVideo?: boolean // Default true | ||
| 43 | }) { | ||
| 44 | const { video, previousVideoState, isNewVideo = true } = options | ||
| 45 | |||
| 46 | return retryTransactionWrapper(() => { | ||
| 47 | return sequelizeTypescript.transaction(async t => { | ||
| 48 | // Maybe the video changed in database, refresh it | ||
| 49 | const videoDatabase = await VideoModel.loadFull(video.uuid, t) | ||
| 50 | // Video does not exist anymore | ||
| 51 | if (!videoDatabase) return undefined | ||
| 52 | |||
| 53 | // Already in its final state | ||
| 54 | if (videoDatabase.state === VideoState.PUBLISHED) { | ||
| 55 | return federateVideoIfNeeded(videoDatabase, false, t) | ||
| 56 | } | ||
| 57 | |||
| 58 | const newState = buildNextVideoState(videoDatabase.state) | ||
| 59 | |||
| 60 | if (newState === VideoState.PUBLISHED) { | ||
| 61 | return moveToPublishedState({ video: videoDatabase, previousVideoState, isNewVideo, transaction: t }) | ||
| 62 | } | ||
| 63 | |||
| 64 | if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
| 65 | return moveToExternalStorageState({ video: videoDatabase, isNewVideo, transaction: t }) | ||
| 66 | } | ||
| 67 | }) | ||
| 68 | }) | ||
| 69 | } | ||
| 70 | |||
| 71 | async function moveToExternalStorageState (options: { | ||
| 72 | video: MVideoFullLight | ||
| 73 | isNewVideo: boolean | ||
| 74 | transaction: Transaction | ||
| 75 | }) { | ||
| 76 | const { video, isNewVideo, transaction } = options | ||
| 77 | |||
| 78 | const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) | ||
| 79 | const pendingTranscode = videoJobInfo?.pendingTranscode || 0 | ||
| 80 | |||
| 81 | // We want to wait all transcoding jobs before moving the video on an external storage | ||
| 82 | if (pendingTranscode !== 0) return false | ||
| 83 | |||
| 84 | const previousVideoState = video.state | ||
| 85 | |||
| 86 | if (video.state !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
| 87 | await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction) | ||
| 88 | } | ||
| 89 | |||
| 90 | logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
| 91 | |||
| 92 | try { | ||
| 93 | await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState, isNewVideo })) | ||
| 94 | |||
| 95 | return true | ||
| 96 | } catch (err) { | ||
| 97 | logger.error('Cannot add move to object storage job', { err }) | ||
| 98 | |||
| 99 | return false | ||
| 100 | } | ||
| 101 | } | ||
| 102 | |||
| 103 | function moveToFailedTranscodingState (video: MVideo) { | ||
| 104 | if (video.state === VideoState.TRANSCODING_FAILED) return | ||
| 105 | |||
| 106 | return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) | ||
| 107 | } | ||
| 108 | |||
| 109 | function moveToFailedMoveToObjectStorageState (video: MVideo) { | ||
| 110 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return | ||
| 111 | |||
| 112 | return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined) | ||
| 113 | } | ||
| 114 | |||
| 115 | // --------------------------------------------------------------------------- | ||
| 116 | |||
| 117 | export { | ||
| 118 | buildNextVideoState, | ||
| 119 | moveToExternalStorageState, | ||
| 120 | moveToFailedTranscodingState, | ||
| 121 | moveToFailedMoveToObjectStorageState, | ||
| 122 | moveToNextState | ||
| 123 | } | ||
| 124 | |||
| 125 | // --------------------------------------------------------------------------- | ||
| 126 | |||
| 127 | async function moveToPublishedState (options: { | ||
| 128 | video: MVideoFullLight | ||
| 129 | isNewVideo: boolean | ||
| 130 | transaction: Transaction | ||
| 131 | previousVideoState?: VideoState | ||
| 132 | }) { | ||
| 133 | const { video, isNewVideo, transaction, previousVideoState } = options | ||
| 134 | const previousState = previousVideoState ?? video.state | ||
| 135 | |||
| 136 | logger.info('Publishing video %s.', video.uuid, { isNewVideo, previousState, tags: [ video.uuid ] }) | ||
| 137 | |||
| 138 | await video.setNewState(VideoState.PUBLISHED, isNewVideo, transaction) | ||
| 139 | |||
| 140 | await federateVideoIfNeeded(video, isNewVideo, transaction) | ||
| 141 | |||
| 142 | if (previousState === VideoState.TO_EDIT) { | ||
| 143 | Notifier.Instance.notifyOfFinishedVideoStudioEdition(video) | ||
| 144 | return | ||
| 145 | } | ||
| 146 | |||
| 147 | if (isNewVideo) { | ||
| 148 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
| 149 | |||
| 150 | if (previousState === VideoState.TO_TRANSCODE) { | ||
| 151 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) | ||
| 152 | } | ||
| 153 | } | ||
| 154 | } | ||
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts deleted file mode 100644 index f549a7084..000000000 --- a/server/lib/video-studio.ts +++ /dev/null | |||
| @@ -1,130 +0,0 @@ | |||
| 1 | import { move, remove } from 'fs-extra' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 4 | import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' | ||
| 5 | import { CONFIG } from '@server/initializers/config' | ||
| 6 | import { UserModel } from '@server/models/user/user' | ||
| 7 | import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models' | ||
| 8 | import { getVideoStreamDuration } from '@shared/ffmpeg' | ||
| 9 | import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@shared/models' | ||
| 10 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
| 11 | import { JobQueue } from './job-queue' | ||
| 12 | import { VideoStudioTranscodingJobHandler } from './runners' | ||
| 13 | import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job' | ||
| 14 | import { getTranscodingJobPriority } from './transcoding/transcoding-priority' | ||
| 15 | import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file' | ||
| 16 | import { VideoPathManager } from './video-path-manager' | ||
| 17 | |||
| 18 | const lTags = loggerTagsFactory('video-studio') | ||
| 19 | |||
| 20 | export function buildTaskFileFieldname (indice: number, fieldName = 'file') { | ||
| 21 | return `tasks[${indice}][options][${fieldName}]` | ||
| 22 | } | ||
| 23 | |||
| 24 | export function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { | ||
| 25 | return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) | ||
| 26 | } | ||
| 27 | |||
| 28 | export function getStudioTaskFilePath (filename: string) { | ||
| 29 | return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, filename) | ||
| 30 | } | ||
| 31 | |||
| 32 | export async function safeCleanupStudioTMPFiles (tasks: VideoStudioTaskPayload[]) { | ||
| 33 | logger.info('Removing studio task files', { tasks, ...lTags() }) | ||
| 34 | |||
| 35 | for (const task of tasks) { | ||
| 36 | try { | ||
| 37 | if (task.name === 'add-intro' || task.name === 'add-outro') { | ||
| 38 | await remove(task.options.file) | ||
| 39 | } else if (task.name === 'add-watermark') { | ||
| 40 | await remove(task.options.file) | ||
| 41 | } | ||
| 42 | } catch (err) { | ||
| 43 | logger.error('Cannot remove studio file', { err }) | ||
| 44 | } | ||
| 45 | } | ||
| 46 | } | ||
| 47 | |||
| 48 | // --------------------------------------------------------------------------- | ||
| 49 | |||
| 50 | export async function approximateIntroOutroAdditionalSize ( | ||
| 51 | video: MVideoFullLight, | ||
| 52 | tasks: VideoStudioTask[], | ||
| 53 | fileFinder: (i: number) => string | ||
| 54 | ) { | ||
| 55 | let additionalDuration = 0 | ||
| 56 | |||
| 57 | for (let i = 0; i < tasks.length; i++) { | ||
| 58 | const task = tasks[i] | ||
| 59 | |||
| 60 | if (task.name !== 'add-intro' && task.name !== 'add-outro') continue | ||
| 61 | |||
| 62 | const filePath = fileFinder(i) | ||
| 63 | additionalDuration += await getVideoStreamDuration(filePath) | ||
| 64 | } | ||
| 65 | |||
| 66 | return (video.getMaxQualityFile().size / video.duration) * additionalDuration | ||
| 67 | } | ||
| 68 | |||
| 69 | // --------------------------------------------------------------------------- | ||
| 70 | |||
| 71 | export async function createVideoStudioJob (options: { | ||
| 72 | video: MVideo | ||
| 73 | user: MUser | ||
| 74 | payload: VideoStudioEditionPayload | ||
| 75 | }) { | ||
| 76 | const { video, user, payload } = options | ||
| 77 | |||
| 78 | const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 }) | ||
| 79 | |||
| 80 | if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) { | ||
| 81 | await new VideoStudioTranscodingJobHandler().create({ video, tasks: payload.tasks, priority }) | ||
| 82 | return | ||
| 83 | } | ||
| 84 | |||
| 85 | await JobQueue.Instance.createJob({ type: 'video-studio-edition', payload, priority }) | ||
| 86 | } | ||
| 87 | |||
| 88 | export async function onVideoStudioEnded (options: { | ||
| 89 | editionResultPath: string | ||
| 90 | tasks: VideoStudioTaskPayload[] | ||
| 91 | video: MVideoFullLight | ||
| 92 | }) { | ||
| 93 | const { video, tasks, editionResultPath } = options | ||
| 94 | |||
| 95 | const newFile = await buildNewFile({ path: editionResultPath, mode: 'web-video' }) | ||
| 96 | newFile.videoId = video.id | ||
| 97 | |||
| 98 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) | ||
| 99 | await move(editionResultPath, outputPath) | ||
| 100 | |||
| 101 | await safeCleanupStudioTMPFiles(tasks) | ||
| 102 | |||
| 103 | await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) | ||
| 104 | await removeAllFiles(video, newFile) | ||
| 105 | |||
| 106 | await newFile.save() | ||
| 107 | |||
| 108 | video.duration = await getVideoStreamDuration(outputPath) | ||
| 109 | await video.save() | ||
| 110 | |||
| 111 | await federateVideoIfNeeded(video, false, undefined) | ||
| 112 | |||
| 113 | const user = await UserModel.loadByVideoId(video.id) | ||
| 114 | |||
| 115 | await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) | ||
| 116 | } | ||
| 117 | |||
| 118 | // --------------------------------------------------------------------------- | ||
| 119 | // Private | ||
| 120 | // --------------------------------------------------------------------------- | ||
| 121 | |||
| 122 | async function removeAllFiles (video: MVideoWithAllFiles, webVideoFileException: MVideoFile) { | ||
| 123 | await removeHLSPlaylist(video) | ||
| 124 | |||
| 125 | for (const file of video.VideoFiles) { | ||
| 126 | if (file.id === webVideoFileException.id) continue | ||
| 127 | |||
| 128 | await removeWebVideoFile(video, file.id) | ||
| 129 | } | ||
| 130 | } | ||
diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts deleted file mode 100644 index e28e55cf7..000000000 --- a/server/lib/video-tokens-manager.ts +++ /dev/null | |||
| @@ -1,78 +0,0 @@ | |||
| 1 | import { LRUCache } from 'lru-cache' | ||
| 2 | import { LRU_CACHE } from '@server/initializers/constants' | ||
| 3 | import { MUserAccountUrl } from '@server/types/models' | ||
| 4 | import { pick } from '@shared/core-utils' | ||
| 5 | import { buildUUID } from '@shared/extra-utils' | ||
| 6 | |||
| 7 | // --------------------------------------------------------------------------- | ||
| 8 | // Create temporary tokens that can be used as URL query parameters to access video static files | ||
| 9 | // --------------------------------------------------------------------------- | ||
| 10 | |||
| 11 | class VideoTokensManager { | ||
| 12 | |||
| 13 | private static instance: VideoTokensManager | ||
| 14 | |||
| 15 | private readonly lruCache = new LRUCache<string, { videoUUID: string, user?: MUserAccountUrl }>({ | ||
| 16 | max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE, | ||
| 17 | ttl: LRU_CACHE.VIDEO_TOKENS.TTL | ||
| 18 | }) | ||
| 19 | |||
| 20 | private constructor () {} | ||
| 21 | |||
| 22 | createForAuthUser (options: { | ||
| 23 | user: MUserAccountUrl | ||
| 24 | videoUUID: string | ||
| 25 | }) { | ||
| 26 | const { token, expires } = this.generateVideoToken() | ||
| 27 | |||
| 28 | this.lruCache.set(token, pick(options, [ 'user', 'videoUUID' ])) | ||
| 29 | |||
| 30 | return { token, expires } | ||
| 31 | } | ||
| 32 | |||
| 33 | createForPasswordProtectedVideo (options: { | ||
| 34 | videoUUID: string | ||
| 35 | }) { | ||
| 36 | const { token, expires } = this.generateVideoToken() | ||
| 37 | |||
| 38 | this.lruCache.set(token, pick(options, [ 'videoUUID' ])) | ||
| 39 | |||
| 40 | return { token, expires } | ||
| 41 | } | ||
| 42 | |||
| 43 | hasToken (options: { | ||
| 44 | token: string | ||
| 45 | videoUUID: string | ||
| 46 | }) { | ||
| 47 | const value = this.lruCache.get(options.token) | ||
| 48 | if (!value) return false | ||
| 49 | |||
| 50 | return value.videoUUID === options.videoUUID | ||
| 51 | } | ||
| 52 | |||
| 53 | getUserFromToken (options: { | ||
| 54 | token: string | ||
| 55 | }) { | ||
| 56 | const value = this.lruCache.get(options.token) | ||
| 57 | if (!value) return undefined | ||
| 58 | |||
| 59 | return value.user | ||
| 60 | } | ||
| 61 | |||
| 62 | static get Instance () { | ||
| 63 | return this.instance || (this.instance = new this()) | ||
| 64 | } | ||
| 65 | |||
| 66 | private generateVideoToken () { | ||
| 67 | const token = buildUUID() | ||
| 68 | const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL) | ||
| 69 | |||
| 70 | return { token, expires } | ||
| 71 | } | ||
| 72 | } | ||
| 73 | |||
| 74 | // --------------------------------------------------------------------------- | ||
| 75 | |||
| 76 | export { | ||
| 77 | VideoTokensManager | ||
| 78 | } | ||
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts deleted file mode 100644 index 0597488ad..000000000 --- a/server/lib/video-urls.ts +++ /dev/null | |||
| @@ -1,31 +0,0 @@ | |||
| 1 | |||
| 2 | import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
| 3 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
| 4 | |||
| 5 | // ################## Redundancy ################## | ||
| 6 | |||
| 7 | function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { | ||
| 8 | // Base URL used by our HLS player | ||
| 9 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | ||
| 10 | } | ||
| 11 | |||
| 12 | function generateWebVideoRedundancyUrl (file: MVideoFile) { | ||
| 13 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | ||
| 14 | } | ||
| 15 | |||
| 16 | // ################## Meta data ################## | ||
| 17 | |||
| 18 | function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { | ||
| 19 | const path = '/api/v1/videos/' | ||
| 20 | |||
| 21 | return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id | ||
| 22 | } | ||
| 23 | |||
| 24 | // --------------------------------------------------------------------------- | ||
| 25 | |||
| 26 | export { | ||
| 27 | getLocalVideoFileMetadataUrl, | ||
| 28 | |||
| 29 | generateWebVideoRedundancyUrl, | ||
| 30 | generateHLSRedundancyUrl | ||
| 31 | } | ||
diff --git a/server/lib/video.ts b/server/lib/video.ts deleted file mode 100644 index 362c861a5..000000000 --- a/server/lib/video.ts +++ /dev/null | |||
| @@ -1,189 +0,0 @@ | |||
| 1 | import { UploadFiles } from 'express' | ||
| 2 | import memoizee from 'memoizee' | ||
| 3 | import { Transaction } from 'sequelize/types' | ||
| 4 | import { CONFIG } from '@server/initializers/config' | ||
| 5 | import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' | ||
| 6 | import { TagModel } from '@server/models/video/tag' | ||
| 7 | import { VideoModel } from '@server/models/video/video' | ||
| 8 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
| 9 | import { FilteredModelAttributes } from '@server/types' | ||
| 10 | import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' | ||
| 11 | import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' | ||
| 12 | import { CreateJobArgument, JobQueue } from './job-queue/job-queue' | ||
| 13 | import { updateLocalVideoMiniatureFromExisting } from './thumbnail' | ||
| 14 | import { moveFilesIfPrivacyChanged } from './video-privacy' | ||
| 15 | |||
| 16 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | ||
| 17 | return { | ||
| 18 | name: videoInfo.name, | ||
| 19 | remote: false, | ||
| 20 | category: videoInfo.category, | ||
| 21 | licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, | ||
| 22 | language: videoInfo.language, | ||
| 23 | commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, | ||
| 24 | downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, | ||
| 25 | waitTranscoding: videoInfo.waitTranscoding || false, | ||
| 26 | nsfw: videoInfo.nsfw || false, | ||
| 27 | description: videoInfo.description, | ||
| 28 | support: videoInfo.support, | ||
| 29 | privacy: videoInfo.privacy || VideoPrivacy.PRIVATE, | ||
| 30 | channelId, | ||
| 31 | originallyPublishedAt: videoInfo.originallyPublishedAt | ||
| 32 | ? new Date(videoInfo.originallyPublishedAt) | ||
| 33 | : null | ||
| 34 | } | ||
| 35 | } | ||
| 36 | |||
| 37 | async function buildVideoThumbnailsFromReq (options: { | ||
| 38 | video: MVideoThumbnail | ||
| 39 | files: UploadFiles | ||
| 40 | fallback: (type: ThumbnailType) => Promise<MThumbnail> | ||
| 41 | automaticallyGenerated?: boolean | ||
| 42 | }) { | ||
| 43 | const { video, files, fallback, automaticallyGenerated } = options | ||
| 44 | |||
| 45 | const promises = [ | ||
| 46 | { | ||
| 47 | type: ThumbnailType.MINIATURE, | ||
| 48 | fieldName: 'thumbnailfile' | ||
| 49 | }, | ||
| 50 | { | ||
| 51 | type: ThumbnailType.PREVIEW, | ||
| 52 | fieldName: 'previewfile' | ||
| 53 | } | ||
| 54 | ].map(p => { | ||
| 55 | const fields = files?.[p.fieldName] | ||
| 56 | |||
| 57 | if (fields) { | ||
| 58 | return updateLocalVideoMiniatureFromExisting({ | ||
| 59 | inputPath: fields[0].path, | ||
| 60 | video, | ||
| 61 | type: p.type, | ||
| 62 | automaticallyGenerated: automaticallyGenerated || false | ||
| 63 | }) | ||
| 64 | } | ||
| 65 | |||
| 66 | return fallback(p.type) | ||
| 67 | }) | ||
| 68 | |||
| 69 | return Promise.all(promises) | ||
| 70 | } | ||
| 71 | |||
| 72 | // --------------------------------------------------------------------------- | ||
| 73 | |||
| 74 | async function setVideoTags (options: { | ||
| 75 | video: MVideoTag | ||
| 76 | tags: string[] | ||
| 77 | transaction?: Transaction | ||
| 78 | }) { | ||
| 79 | const { video, tags, transaction } = options | ||
| 80 | |||
| 81 | const internalTags = tags || [] | ||
| 82 | const tagInstances = await TagModel.findOrCreateTags(internalTags, transaction) | ||
| 83 | |||
| 84 | await video.$set('Tags', tagInstances, { transaction }) | ||
| 85 | video.Tags = tagInstances | ||
| 86 | } | ||
| 87 | |||
| 88 | // --------------------------------------------------------------------------- | ||
| 89 | |||
| 90 | async function buildMoveToObjectStorageJob (options: { | ||
| 91 | video: MVideoUUID | ||
| 92 | previousVideoState: VideoState | ||
| 93 | isNewVideo?: boolean // Default true | ||
| 94 | }) { | ||
| 95 | const { video, previousVideoState, isNewVideo = true } = options | ||
| 96 | |||
| 97 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') | ||
| 98 | |||
| 99 | return { | ||
| 100 | type: 'move-to-object-storage' as 'move-to-object-storage', | ||
| 101 | payload: { | ||
| 102 | videoUUID: video.uuid, | ||
| 103 | isNewVideo, | ||
| 104 | previousVideoState | ||
| 105 | } | ||
| 106 | } | ||
| 107 | } | ||
| 108 | |||
| 109 | // --------------------------------------------------------------------------- | ||
| 110 | |||
| 111 | async function getVideoDuration (videoId: number | string) { | ||
| 112 | const video = await VideoModel.load(videoId) | ||
| 113 | |||
| 114 | const duration = video.isLive | ||
| 115 | ? undefined | ||
| 116 | : video.duration | ||
| 117 | |||
| 118 | return { duration, isLive: video.isLive } | ||
| 119 | } | ||
| 120 | |||
| 121 | const getCachedVideoDuration = memoizee(getVideoDuration, { | ||
| 122 | promise: true, | ||
| 123 | max: MEMOIZE_LENGTH.VIDEO_DURATION, | ||
| 124 | maxAge: MEMOIZE_TTL.VIDEO_DURATION | ||
| 125 | }) | ||
| 126 | |||
| 127 | // --------------------------------------------------------------------------- | ||
| 128 | |||
| 129 | async function addVideoJobsAfterUpdate (options: { | ||
| 130 | video: MVideoFullLight | ||
| 131 | isNewVideo: boolean | ||
| 132 | |||
| 133 | nameChanged: boolean | ||
| 134 | oldPrivacy: VideoPrivacy | ||
| 135 | }) { | ||
| 136 | const { video, nameChanged, oldPrivacy, isNewVideo } = options | ||
| 137 | const jobs: CreateJobArgument[] = [] | ||
| 138 | |||
| 139 | const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy) | ||
| 140 | |||
| 141 | if (!video.isLive && (nameChanged || filePathChanged)) { | ||
| 142 | for (const file of (video.VideoFiles || [])) { | ||
| 143 | const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } | ||
| 144 | |||
| 145 | jobs.push({ type: 'manage-video-torrent', payload }) | ||
| 146 | } | ||
| 147 | |||
| 148 | const hls = video.getHLSPlaylist() | ||
| 149 | |||
| 150 | for (const file of (hls?.VideoFiles || [])) { | ||
| 151 | const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } | ||
| 152 | |||
| 153 | jobs.push({ type: 'manage-video-torrent', payload }) | ||
| 154 | } | ||
| 155 | } | ||
| 156 | |||
| 157 | jobs.push({ | ||
| 158 | type: 'federate-video', | ||
| 159 | payload: { | ||
| 160 | videoUUID: video.uuid, | ||
| 161 | isNewVideo | ||
| 162 | } | ||
| 163 | }) | ||
| 164 | |||
| 165 | const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy) | ||
| 166 | |||
| 167 | if (wasConfidentialVideo) { | ||
| 168 | jobs.push({ | ||
| 169 | type: 'notify', | ||
| 170 | payload: { | ||
| 171 | action: 'new-video', | ||
| 172 | videoUUID: video.uuid | ||
| 173 | } | ||
| 174 | }) | ||
| 175 | } | ||
| 176 | |||
| 177 | return JobQueue.Instance.createSequentialJobFlow(...jobs) | ||
| 178 | } | ||
| 179 | |||
| 180 | // --------------------------------------------------------------------------- | ||
| 181 | |||
| 182 | export { | ||
| 183 | buildLocalVideoFromReq, | ||
| 184 | buildVideoThumbnailsFromReq, | ||
| 185 | setVideoTags, | ||
| 186 | buildMoveToObjectStorageJob, | ||
| 187 | addVideoJobsAfterUpdate, | ||
| 188 | getCachedVideoDuration | ||
| 189 | } | ||
diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts deleted file mode 100644 index 139471183..000000000 --- a/server/lib/views/shared/index.ts +++ /dev/null | |||
| @@ -1,3 +0,0 @@ | |||
| 1 | export * from './video-viewer-counters' | ||
| 2 | export * from './video-viewer-stats' | ||
| 3 | export * from './video-views' | ||
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts deleted file mode 100644 index f5b83130e..000000000 --- a/server/lib/views/shared/video-viewer-counters.ts +++ /dev/null | |||
| @@ -1,198 +0,0 @@ | |||
| 1 | import { isTestOrDevInstance } from '@server/helpers/core-utils' | ||
| 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 3 | import { VIEW_LIFETIME } from '@server/initializers/constants' | ||
| 4 | import { sendView } from '@server/lib/activitypub/send/send-view' | ||
| 5 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
| 6 | import { getServerActor } from '@server/models/application/application' | ||
| 7 | import { VideoModel } from '@server/models/video/video' | ||
| 8 | import { MVideo, MVideoImmutable } from '@server/types/models' | ||
| 9 | import { buildUUID, sha256 } from '@shared/extra-utils' | ||
| 10 | |||
| 11 | const lTags = loggerTagsFactory('views') | ||
| 12 | |||
| 13 | export type ViewerScope = 'local' | 'remote' | ||
| 14 | export type VideoScope = 'local' | 'remote' | ||
| 15 | |||
| 16 | type Viewer = { | ||
| 17 | expires: number | ||
| 18 | id: string | ||
| 19 | viewerScope: ViewerScope | ||
| 20 | videoScope: VideoScope | ||
| 21 | lastFederation?: number | ||
| 22 | } | ||
| 23 | |||
| 24 | export class VideoViewerCounters { | ||
| 25 | |||
| 26 | // expires is new Date().getTime() | ||
| 27 | private readonly viewersPerVideo = new Map<number, Viewer[]>() | ||
| 28 | private readonly idToViewer = new Map<string, Viewer>() | ||
| 29 | |||
| 30 | private readonly salt = buildUUID() | ||
| 31 | |||
| 32 | private processingViewerCounters = false | ||
| 33 | |||
| 34 | constructor () { | ||
| 35 | setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER) | ||
| 36 | } | ||
| 37 | |||
| 38 | // --------------------------------------------------------------------------- | ||
| 39 | |||
| 40 | async addLocalViewer (options: { | ||
| 41 | video: MVideoImmutable | ||
| 42 | ip: string | ||
| 43 | }) { | ||
| 44 | const { video, ip } = options | ||
| 45 | |||
| 46 | logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) }) | ||
| 47 | |||
| 48 | const viewerId = this.generateViewerId(ip, video.uuid) | ||
| 49 | const viewer = this.idToViewer.get(viewerId) | ||
| 50 | |||
| 51 | if (viewer) { | ||
| 52 | viewer.expires = this.buildViewerExpireTime() | ||
| 53 | await this.federateViewerIfNeeded(video, viewer) | ||
| 54 | |||
| 55 | return false | ||
| 56 | } | ||
| 57 | |||
| 58 | const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' }) | ||
| 59 | await this.federateViewerIfNeeded(video, newViewer) | ||
| 60 | |||
| 61 | return true | ||
| 62 | } | ||
| 63 | |||
| 64 | async addRemoteViewer (options: { | ||
| 65 | video: MVideo | ||
| 66 | viewerId: string | ||
| 67 | viewerExpires: Date | ||
| 68 | }) { | ||
| 69 | const { video, viewerExpires, viewerId } = options | ||
| 70 | |||
| 71 | logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) | ||
| 72 | |||
| 73 | await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' }) | ||
| 74 | |||
| 75 | return true | ||
| 76 | } | ||
| 77 | |||
| 78 | // --------------------------------------------------------------------------- | ||
| 79 | |||
| 80 | getTotalViewers (options: { | ||
| 81 | viewerScope: ViewerScope | ||
| 82 | videoScope: VideoScope | ||
| 83 | }) { | ||
| 84 | let total = 0 | ||
| 85 | |||
| 86 | for (const viewers of this.viewersPerVideo.values()) { | ||
| 87 | total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length | ||
| 88 | } | ||
| 89 | |||
| 90 | return total | ||
| 91 | } | ||
| 92 | |||
| 93 | getViewers (video: MVideo) { | ||
| 94 | const viewers = this.viewersPerVideo.get(video.id) | ||
| 95 | if (!viewers) return 0 | ||
| 96 | |||
| 97 | return viewers.length | ||
| 98 | } | ||
| 99 | |||
| 100 | buildViewerExpireTime () { | ||
| 101 | return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER | ||
| 102 | } | ||
| 103 | |||
| 104 | // --------------------------------------------------------------------------- | ||
| 105 | |||
| 106 | private async addViewerToVideo (options: { | ||
| 107 | video: MVideoImmutable | ||
| 108 | viewerId: string | ||
| 109 | viewerScope: ViewerScope | ||
| 110 | viewerExpires?: Date | ||
| 111 | }) { | ||
| 112 | const { video, viewerExpires, viewerId, viewerScope } = options | ||
| 113 | |||
| 114 | let watchers = this.viewersPerVideo.get(video.id) | ||
| 115 | |||
| 116 | if (!watchers) { | ||
| 117 | watchers = [] | ||
| 118 | this.viewersPerVideo.set(video.id, watchers) | ||
| 119 | } | ||
| 120 | |||
| 121 | const expires = viewerExpires | ||
| 122 | ? viewerExpires.getTime() | ||
| 123 | : this.buildViewerExpireTime() | ||
| 124 | |||
| 125 | const videoScope: VideoScope = video.remote | ||
| 126 | ? 'remote' | ||
| 127 | : 'local' | ||
| 128 | |||
| 129 | const viewer = { id: viewerId, expires, videoScope, viewerScope } | ||
| 130 | watchers.push(viewer) | ||
| 131 | |||
| 132 | this.idToViewer.set(viewerId, viewer) | ||
| 133 | |||
| 134 | await this.notifyClients(video.id, watchers.length) | ||
| 135 | |||
| 136 | return viewer | ||
| 137 | } | ||
| 138 | |||
| 139 | private async cleanViewerCounters () { | ||
| 140 | if (this.processingViewerCounters) return | ||
| 141 | this.processingViewerCounters = true | ||
| 142 | |||
| 143 | if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags()) | ||
| 144 | |||
| 145 | try { | ||
| 146 | for (const videoId of this.viewersPerVideo.keys()) { | ||
| 147 | const notBefore = new Date().getTime() | ||
| 148 | |||
| 149 | const viewers = this.viewersPerVideo.get(videoId) | ||
| 150 | |||
| 151 | // Only keep not expired viewers | ||
| 152 | const newViewers: Viewer[] = [] | ||
| 153 | |||
| 154 | // Filter new viewers | ||
| 155 | for (const viewer of viewers) { | ||
| 156 | if (viewer.expires > notBefore) { | ||
| 157 | newViewers.push(viewer) | ||
| 158 | } else { | ||
| 159 | this.idToViewer.delete(viewer.id) | ||
| 160 | } | ||
| 161 | } | ||
| 162 | |||
| 163 | if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) | ||
| 164 | else this.viewersPerVideo.set(videoId, newViewers) | ||
| 165 | |||
| 166 | await this.notifyClients(videoId, newViewers.length) | ||
| 167 | } | ||
| 168 | } catch (err) { | ||
| 169 | logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) | ||
| 170 | } | ||
| 171 | |||
| 172 | this.processingViewerCounters = false | ||
| 173 | } | ||
| 174 | |||
| 175 | private async notifyClients (videoId: string | number, viewersLength: number) { | ||
| 176 | const video = await VideoModel.loadImmutableAttributes(videoId) | ||
| 177 | if (!video) return | ||
| 178 | |||
| 179 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) | ||
| 180 | |||
| 181 | logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) | ||
| 182 | } | ||
| 183 | |||
| 184 | private generateViewerId (ip: string, videoUUID: string) { | ||
| 185 | return sha256(this.salt + '-' + ip + '-' + videoUUID) | ||
| 186 | } | ||
| 187 | |||
| 188 | private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) { | ||
| 189 | // Federate the viewer if it's been a "long" time we did not | ||
| 190 | const now = new Date().getTime() | ||
| 191 | const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) | ||
| 192 | |||
| 193 | if (viewer.lastFederation && viewer.lastFederation > federationLimit) return | ||
| 194 | |||
| 195 | await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id }) | ||
| 196 | viewer.lastFederation = now | ||
| 197 | } | ||
| 198 | } | ||
diff --git a/server/lib/views/shared/video-viewer-stats.ts b/server/lib/views/shared/video-viewer-stats.ts deleted file mode 100644 index ebd963e59..000000000 --- a/server/lib/views/shared/video-viewer-stats.ts +++ /dev/null | |||
| @@ -1,196 +0,0 @@ | |||
| 1 | import { Transaction } from 'sequelize/types' | ||
| 2 | import { isTestOrDevInstance } from '@server/helpers/core-utils' | ||
| 3 | import { GeoIP } from '@server/helpers/geo-ip' | ||
| 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 5 | import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants' | ||
| 6 | import { sequelizeTypescript } from '@server/initializers/database' | ||
| 7 | import { sendCreateWatchAction } from '@server/lib/activitypub/send' | ||
| 8 | import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url' | ||
| 9 | import { Redis } from '@server/lib/redis' | ||
| 10 | import { VideoModel } from '@server/models/video/video' | ||
| 11 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
| 12 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' | ||
| 13 | import { MVideo, MVideoImmutable } from '@server/types/models' | ||
| 14 | import { VideoViewEvent } from '@shared/models' | ||
| 15 | |||
| 16 | const lTags = loggerTagsFactory('views') | ||
| 17 | |||
| 18 | type LocalViewerStats = { | ||
| 19 | firstUpdated: number // Date.getTime() | ||
| 20 | lastUpdated: number // Date.getTime() | ||
| 21 | |||
| 22 | watchSections: { | ||
| 23 | start: number | ||
| 24 | end: number | ||
| 25 | }[] | ||
| 26 | |||
| 27 | watchTime: number | ||
| 28 | |||
| 29 | country: string | ||
| 30 | |||
| 31 | videoId: number | ||
| 32 | } | ||
| 33 | |||
| 34 | export class VideoViewerStats { | ||
| 35 | private processingViewersStats = false | ||
| 36 | |||
| 37 | constructor () { | ||
| 38 | setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) | ||
| 39 | } | ||
| 40 | |||
| 41 | // --------------------------------------------------------------------------- | ||
| 42 | |||
| 43 | async addLocalViewer (options: { | ||
| 44 | video: MVideoImmutable | ||
| 45 | currentTime: number | ||
| 46 | ip: string | ||
| 47 | viewEvent?: VideoViewEvent | ||
| 48 | }) { | ||
| 49 | const { video, ip, viewEvent, currentTime } = options | ||
| 50 | |||
| 51 | logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) | ||
| 52 | |||
| 53 | return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) | ||
| 54 | } | ||
| 55 | |||
| 56 | // --------------------------------------------------------------------------- | ||
| 57 | |||
| 58 | async getWatchTime (videoId: number, ip: string) { | ||
| 59 | const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) | ||
| 60 | |||
| 61 | return stats?.watchTime || 0 | ||
| 62 | } | ||
| 63 | |||
| 64 | // --------------------------------------------------------------------------- | ||
| 65 | |||
| 66 | private async updateLocalViewerStats (options: { | ||
| 67 | video: MVideoImmutable | ||
| 68 | ip: string | ||
| 69 | currentTime: number | ||
| 70 | viewEvent?: VideoViewEvent | ||
| 71 | }) { | ||
| 72 | const { video, ip, viewEvent, currentTime } = options | ||
| 73 | const nowMs = new Date().getTime() | ||
| 74 | |||
| 75 | let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id }) | ||
| 76 | |||
| 77 | if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) { | ||
| 78 | logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) }) | ||
| 79 | return | ||
| 80 | } | ||
| 81 | |||
| 82 | if (!stats) { | ||
| 83 | const country = await GeoIP.Instance.safeCountryISOLookup(ip) | ||
| 84 | |||
| 85 | stats = { | ||
| 86 | firstUpdated: nowMs, | ||
| 87 | lastUpdated: nowMs, | ||
| 88 | |||
| 89 | watchSections: [], | ||
| 90 | |||
| 91 | watchTime: 0, | ||
| 92 | |||
| 93 | country, | ||
| 94 | videoId: video.id | ||
| 95 | } | ||
| 96 | } | ||
| 97 | |||
| 98 | stats.lastUpdated = nowMs | ||
| 99 | |||
| 100 | if (viewEvent === 'seek' || stats.watchSections.length === 0) { | ||
| 101 | stats.watchSections.push({ | ||
| 102 | start: currentTime, | ||
| 103 | end: currentTime | ||
| 104 | }) | ||
| 105 | } else { | ||
| 106 | const lastSection = stats.watchSections[stats.watchSections.length - 1] | ||
| 107 | |||
| 108 | if (lastSection.start > currentTime) { | ||
| 109 | logger.debug('Invalid end watch section %d. Last start record was at %d. Starting a new section.', currentTime, lastSection.start) | ||
| 110 | |||
| 111 | stats.watchSections.push({ | ||
| 112 | start: currentTime, | ||
| 113 | end: currentTime | ||
| 114 | }) | ||
| 115 | } else { | ||
| 116 | lastSection.end = currentTime | ||
| 117 | } | ||
| 118 | } | ||
| 119 | |||
| 120 | stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections) | ||
| 121 | |||
| 122 | logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) | ||
| 123 | |||
| 124 | await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) | ||
| 125 | } | ||
| 126 | |||
| 127 | async processViewerStats () { | ||
| 128 | if (this.processingViewersStats) return | ||
| 129 | this.processingViewersStats = true | ||
| 130 | |||
| 131 | if (!isTestOrDevInstance()) logger.info('Processing viewer statistics.', lTags()) | ||
| 132 | |||
| 133 | const now = new Date().getTime() | ||
| 134 | |||
| 135 | try { | ||
| 136 | const allKeys = await Redis.Instance.listLocalVideoViewerKeys() | ||
| 137 | |||
| 138 | for (const key of allKeys) { | ||
| 139 | const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key }) | ||
| 140 | |||
| 141 | // Process expired stats | ||
| 142 | if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) { | ||
| 143 | continue | ||
| 144 | } | ||
| 145 | |||
| 146 | try { | ||
| 147 | await sequelizeTypescript.transaction(async t => { | ||
| 148 | const video = await VideoModel.load(stats.videoId, t) | ||
| 149 | if (!video) return | ||
| 150 | |||
| 151 | const statsModel = await this.saveViewerStats(video, stats, t) | ||
| 152 | |||
| 153 | if (video.remote) { | ||
| 154 | await sendCreateWatchAction(statsModel, t) | ||
| 155 | } | ||
| 156 | }) | ||
| 157 | |||
| 158 | await Redis.Instance.deleteLocalVideoViewersKeys(key) | ||
| 159 | } catch (err) { | ||
| 160 | logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() }) | ||
| 161 | } | ||
| 162 | } | ||
| 163 | } catch (err) { | ||
| 164 | logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) | ||
| 165 | } | ||
| 166 | |||
| 167 | this.processingViewersStats = false | ||
| 168 | } | ||
| 169 | |||
| 170 | private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { | ||
| 171 | const statsModel = new LocalVideoViewerModel({ | ||
| 172 | startDate: new Date(stats.firstUpdated), | ||
| 173 | endDate: new Date(stats.lastUpdated), | ||
| 174 | watchTime: stats.watchTime, | ||
| 175 | country: stats.country, | ||
| 176 | videoId: video.id | ||
| 177 | }) | ||
| 178 | |||
| 179 | statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel) | ||
| 180 | statsModel.Video = video as VideoModel | ||
| 181 | |||
| 182 | await statsModel.save({ transaction }) | ||
| 183 | |||
| 184 | statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({ | ||
| 185 | localVideoViewerId: statsModel.id, | ||
| 186 | watchSections: stats.watchSections, | ||
| 187 | transaction | ||
| 188 | }) | ||
| 189 | |||
| 190 | return statsModel | ||
| 191 | } | ||
| 192 | |||
| 193 | private buildWatchTimeFromSections (sections: { start: number, end: number }[]) { | ||
| 194 | return sections.reduce((p, current) => p + (current.end - current.start), 0) | ||
| 195 | } | ||
| 196 | } | ||
diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts deleted file mode 100644 index e563287e1..000000000 --- a/server/lib/views/shared/video-views.ts +++ /dev/null | |||
| @@ -1,70 +0,0 @@ | |||
| 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 2 | import { sendView } from '@server/lib/activitypub/send/send-view' | ||
| 3 | import { getCachedVideoDuration } from '@server/lib/video' | ||
| 4 | import { getServerActor } from '@server/models/application/application' | ||
| 5 | import { MVideo, MVideoImmutable } from '@server/types/models' | ||
| 6 | import { buildUUID } from '@shared/extra-utils' | ||
| 7 | import { Redis } from '../../redis' | ||
| 8 | |||
| 9 | const lTags = loggerTagsFactory('views') | ||
| 10 | |||
| 11 | export class VideoViews { | ||
| 12 | |||
| 13 | async addLocalView (options: { | ||
| 14 | video: MVideoImmutable | ||
| 15 | ip: string | ||
| 16 | watchTime: number | ||
| 17 | }) { | ||
| 18 | const { video, ip, watchTime } = options | ||
| 19 | |||
| 20 | logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) | ||
| 21 | |||
| 22 | if (!await this.hasEnoughWatchTime(video, watchTime)) return false | ||
| 23 | |||
| 24 | const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) | ||
| 25 | if (viewExists) return false | ||
| 26 | |||
| 27 | await Redis.Instance.setIPVideoView(ip, video.uuid) | ||
| 28 | |||
| 29 | await this.addView(video) | ||
| 30 | |||
| 31 | await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() }) | ||
| 32 | |||
| 33 | return true | ||
| 34 | } | ||
| 35 | |||
| 36 | async addRemoteView (options: { | ||
| 37 | video: MVideo | ||
| 38 | }) { | ||
| 39 | const { video } = options | ||
| 40 | |||
| 41 | logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) }) | ||
| 42 | |||
| 43 | await this.addView(video) | ||
| 44 | |||
| 45 | return true | ||
| 46 | } | ||
| 47 | |||
| 48 | // --------------------------------------------------------------------------- | ||
| 49 | |||
| 50 | private async addView (video: MVideoImmutable) { | ||
| 51 | const promises: Promise<any>[] = [] | ||
| 52 | |||
| 53 | if (video.isOwned()) { | ||
| 54 | promises.push(Redis.Instance.addLocalVideoView(video.id)) | ||
| 55 | } | ||
| 56 | |||
| 57 | promises.push(Redis.Instance.addVideoViewStats(video.id)) | ||
| 58 | |||
| 59 | await Promise.all(promises) | ||
| 60 | } | ||
| 61 | |||
| 62 | private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) { | ||
| 63 | const { duration, isLive } = await getCachedVideoDuration(video.id) | ||
| 64 | |||
| 65 | if (isLive || duration >= 30) return watchTime >= 30 | ||
| 66 | |||
| 67 | // Check more than 50% of the video is watched | ||
| 68 | return duration / watchTime < 2 | ||
| 69 | } | ||
| 70 | } | ||
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts deleted file mode 100644 index c088dad5e..000000000 --- a/server/lib/views/video-views-manager.ts +++ /dev/null | |||
| @@ -1,100 +0,0 @@ | |||
| 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
| 2 | import { MVideo, MVideoImmutable } from '@server/types/models' | ||
| 3 | import { VideoViewEvent } from '@shared/models' | ||
| 4 | import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared' | ||
| 5 | |||
| 6 | /** | ||
| 7 | * If processing a local view: | ||
| 8 | * - We update viewer information (segments watched, watch time etc) | ||
| 9 | * - We add +1 to video viewers counter if this is a new viewer | ||
| 10 | * - We add +1 to video views counter if this is a new view and if the user watched enough seconds | ||
| 11 | * - We send AP message to notify about this viewer and this view | ||
| 12 | * - We update last video time for the user if authenticated | ||
| 13 | * | ||
| 14 | * If processing a remote view: | ||
| 15 | * - We add +1 to video viewers counter | ||
| 16 | * - We add +1 to video views counter | ||
| 17 | * | ||
| 18 | * A viewer is a someone that watched one or multiple sections of a video | ||
| 19 | * A viewer that watched only a few seconds of a video may not increment the video views counter | ||
| 20 | * Viewers statistics are sent to origin instance using the `WatchAction` ActivityPub object | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | const lTags = loggerTagsFactory('views') | ||
| 25 | |||
| 26 | export class VideoViewsManager { | ||
| 27 | |||
| 28 | private static instance: VideoViewsManager | ||
| 29 | |||
| 30 | private videoViewerStats: VideoViewerStats | ||
| 31 | private videoViewerCounters: VideoViewerCounters | ||
| 32 | private videoViews: VideoViews | ||
| 33 | |||
| 34 | private constructor () { | ||
| 35 | } | ||
| 36 | |||
| 37 | init () { | ||
| 38 | this.videoViewerStats = new VideoViewerStats() | ||
| 39 | this.videoViewerCounters = new VideoViewerCounters() | ||
| 40 | this.videoViews = new VideoViews() | ||
| 41 | } | ||
| 42 | |||
| 43 | async processLocalView (options: { | ||
| 44 | video: MVideoImmutable | ||
| 45 | currentTime: number | ||
| 46 | ip: string | null | ||
| 47 | viewEvent?: VideoViewEvent | ||
| 48 | }) { | ||
| 49 | const { video, ip, viewEvent, currentTime } = options | ||
| 50 | |||
| 51 | logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) | ||
| 52 | |||
| 53 | await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime }) | ||
| 54 | |||
| 55 | const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip }) | ||
| 56 | |||
| 57 | // Do it after added local viewer to fetch updated information | ||
| 58 | const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip) | ||
| 59 | |||
| 60 | const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) | ||
| 61 | |||
| 62 | return { successView, successViewer } | ||
| 63 | } | ||
| 64 | |||
| 65 | async processRemoteView (options: { | ||
| 66 | video: MVideo | ||
| 67 | viewerId: string | null | ||
| 68 | viewerExpires?: Date | ||
| 69 | }) { | ||
| 70 | const { video, viewerId, viewerExpires } = options | ||
| 71 | |||
| 72 | logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() }) | ||
| 73 | |||
| 74 | if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires }) | ||
| 75 | else await this.videoViews.addRemoteView({ video }) | ||
| 76 | } | ||
| 77 | |||
| 78 | getViewers (video: MVideo) { | ||
| 79 | return this.videoViewerCounters.getViewers(video) | ||
| 80 | } | ||
| 81 | |||
| 82 | getTotalViewers (options: { | ||
| 83 | viewerScope: ViewerScope | ||
| 84 | videoScope: VideoScope | ||
| 85 | }) { | ||
| 86 | return this.videoViewerCounters.getTotalViewers(options) | ||
| 87 | } | ||
| 88 | |||
| 89 | buildViewerExpireTime () { | ||
| 90 | return this.videoViewerCounters.buildViewerExpireTime() | ||
| 91 | } | ||
| 92 | |||
| 93 | processViewerStats () { | ||
| 94 | return this.videoViewerStats.processViewerStats() | ||
| 95 | } | ||
| 96 | |||
| 97 | static get Instance () { | ||
| 98 | return this.instance || (this.instance = new this()) | ||
| 99 | } | ||
| 100 | } | ||
diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts deleted file mode 100644 index 48b6c682b..000000000 --- a/server/lib/worker/parent-process.ts +++ /dev/null | |||
| @@ -1,77 +0,0 @@ | |||
| 1 | import { join } from 'path' | ||
| 2 | import Piscina from 'piscina' | ||
| 3 | import { processImage } from '@server/helpers/image-utils' | ||
| 4 | import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants' | ||
| 5 | import { httpBroadcast } from './workers/http-broadcast' | ||
| 6 | import { downloadImage } from './workers/image-downloader' | ||
| 7 | |||
| 8 | let downloadImageWorker: Piscina | ||
| 9 | |||
| 10 | function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> { | ||
| 11 | if (!downloadImageWorker) { | ||
| 12 | downloadImageWorker = new Piscina({ | ||
| 13 | filename: join(__dirname, 'workers', 'image-downloader.js'), | ||
| 14 | concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY, | ||
| 15 | maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS | ||
| 16 | }) | ||
| 17 | } | ||
| 18 | |||
| 19 | return downloadImageWorker.run(options) | ||
| 20 | } | ||
| 21 | |||
| 22 | // --------------------------------------------------------------------------- | ||
| 23 | |||
| 24 | let processImageWorker: Piscina | ||
| 25 | |||
| 26 | function processImageFromWorker (options: Parameters<typeof processImage>[0]): Promise<ReturnType<typeof processImage>> { | ||
| 27 | if (!processImageWorker) { | ||
| 28 | processImageWorker = new Piscina({ | ||
| 29 | filename: join(__dirname, 'workers', 'image-processor.js'), | ||
| 30 | concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY, | ||
| 31 | maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS | ||
| 32 | }) | ||
| 33 | } | ||
| 34 | |||
| 35 | return processImageWorker.run(options) | ||
| 36 | } | ||
| 37 | |||
| 38 | // --------------------------------------------------------------------------- | ||
| 39 | |||
| 40 | let parallelHTTPBroadcastWorker: Piscina | ||
| 41 | |||
| 42 | function parallelHTTPBroadcastFromWorker (options: Parameters<typeof httpBroadcast>[0]): Promise<ReturnType<typeof httpBroadcast>> { | ||
| 43 | if (!parallelHTTPBroadcastWorker) { | ||
| 44 | parallelHTTPBroadcastWorker = new Piscina({ | ||
| 45 | filename: join(__dirname, 'workers', 'http-broadcast.js'), | ||
| 46 | // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs | ||
| 47 | concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'], | ||
| 48 | maxThreads: 1 | ||
| 49 | }) | ||
| 50 | } | ||
| 51 | |||
| 52 | return parallelHTTPBroadcastWorker.run(options) | ||
| 53 | } | ||
| 54 | |||
| 55 | // --------------------------------------------------------------------------- | ||
| 56 | |||
| 57 | let sequentialHTTPBroadcastWorker: Piscina | ||
| 58 | |||
| 59 | function sequentialHTTPBroadcastFromWorker (options: Parameters<typeof httpBroadcast>[0]): Promise<ReturnType<typeof httpBroadcast>> { | ||
| 60 | if (!sequentialHTTPBroadcastWorker) { | ||
| 61 | sequentialHTTPBroadcastWorker = new Piscina({ | ||
| 62 | filename: join(__dirname, 'workers', 'http-broadcast.js'), | ||
| 63 | // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs | ||
| 64 | concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'], | ||
| 65 | maxThreads: 1 | ||
| 66 | }) | ||
| 67 | } | ||
| 68 | |||
| 69 | return sequentialHTTPBroadcastWorker.run(options) | ||
| 70 | } | ||
| 71 | |||
| 72 | export { | ||
| 73 | downloadImageFromWorker, | ||
| 74 | processImageFromWorker, | ||
| 75 | parallelHTTPBroadcastFromWorker, | ||
| 76 | sequentialHTTPBroadcastFromWorker | ||
| 77 | } | ||
diff --git a/server/lib/worker/workers/http-broadcast.ts b/server/lib/worker/workers/http-broadcast.ts deleted file mode 100644 index 8c9c6b8ca..000000000 --- a/server/lib/worker/workers/http-broadcast.ts +++ /dev/null | |||
| @@ -1,32 +0,0 @@ | |||
| 1 | import { map } from 'bluebird' | ||
| 2 | import { logger } from '@server/helpers/logger' | ||
| 3 | import { doRequest, PeerTubeRequestOptions } from '@server/helpers/requests' | ||
| 4 | import { BROADCAST_CONCURRENCY } from '@server/initializers/constants' | ||
| 5 | |||
| 6 | async function httpBroadcast (payload: { | ||
| 7 | uris: string[] | ||
| 8 | requestOptions: PeerTubeRequestOptions | ||
| 9 | }) { | ||
| 10 | const { uris, requestOptions } = payload | ||
| 11 | |||
| 12 | const badUrls: string[] = [] | ||
| 13 | const goodUrls: string[] = [] | ||
| 14 | |||
| 15 | await map(uris, async uri => { | ||
| 16 | try { | ||
| 17 | await doRequest(uri, requestOptions) | ||
| 18 | goodUrls.push(uri) | ||
| 19 | } catch (err) { | ||
| 20 | logger.debug('HTTP broadcast to %s failed.', uri, { err }) | ||
| 21 | badUrls.push(uri) | ||
| 22 | } | ||
| 23 | }, { concurrency: BROADCAST_CONCURRENCY }) | ||
| 24 | |||
| 25 | return { goodUrls, badUrls } | ||
| 26 | } | ||
| 27 | |||
| 28 | module.exports = httpBroadcast | ||
| 29 | |||
| 30 | export { | ||
| 31 | httpBroadcast | ||
| 32 | } | ||
diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts deleted file mode 100644 index 209594589..000000000 --- a/server/lib/worker/workers/image-downloader.ts +++ /dev/null | |||
| @@ -1,35 +0,0 @@ | |||
| 1 | import { remove } from 'fs-extra' | ||
| 2 | import { join } from 'path' | ||
| 3 | import { processImage } from '@server/helpers/image-utils' | ||
| 4 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | ||
| 5 | import { CONFIG } from '@server/initializers/config' | ||
| 6 | |||
| 7 | async function downloadImage (options: { | ||
| 8 | url: string | ||
| 9 | destDir: string | ||
| 10 | destName: string | ||
| 11 | size: { width: number, height: number } | ||
| 12 | }) { | ||
| 13 | const { url, destDir, destName, size } = options | ||
| 14 | |||
| 15 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) | ||
| 16 | await doRequestAndSaveToFile(url, tmpPath) | ||
| 17 | |||
| 18 | const destPath = join(destDir, destName) | ||
| 19 | |||
| 20 | try { | ||
| 21 | await processImage({ path: tmpPath, destination: destPath, newSize: size }) | ||
| 22 | } catch (err) { | ||
| 23 | await remove(tmpPath) | ||
| 24 | |||
| 25 | throw err | ||
| 26 | } | ||
| 27 | |||
| 28 | return destPath | ||
| 29 | } | ||
| 30 | |||
| 31 | module.exports = downloadImage | ||
| 32 | |||
| 33 | export { | ||
| 34 | downloadImage | ||
| 35 | } | ||
diff --git a/server/lib/worker/workers/image-processor.ts b/server/lib/worker/workers/image-processor.ts deleted file mode 100644 index 0ab41a5a0..000000000 --- a/server/lib/worker/workers/image-processor.ts +++ /dev/null | |||
| @@ -1,7 +0,0 @@ | |||
| 1 | import { processImage } from '@server/helpers/image-utils' | ||
| 2 | |||
| 3 | module.exports = processImage | ||
| 4 | |||
| 5 | export { | ||
| 6 | processImage | ||
| 7 | } | ||