From 505319061e28da647c2e9af6e65721d6b8f9da1b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Oct 2018 10:37:08 +0200 Subject: Fix avatar update --- server/lib/avatar.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server/lib') diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 4b6bc3185..021426a1a 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts @@ -7,10 +7,11 @@ import { AccountModel } from '../models/account/account' import { VideoChannelModel } from '../models/video/video-channel' import { extname, join } from 'path' import { retryTransactionWrapper } from '../helpers/database-utils' +import * as uuidv4 from 'uuid/v4' async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { const extension = extname(avatarPhysicalFile.filename) - const avatarName = accountOrChannel.Actor.uuid + extension + const avatarName = uuidv4() + extension const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) -- cgit v1.2.3 From ecf3f060ef8e40846ee41c9dcdf288065f4c461d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Oct 2018 10:37:08 +0200 Subject: Fix avatar update --- server/lib/avatar.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server/lib') diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 4b6bc3185..021426a1a 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts @@ -7,10 +7,11 @@ import { AccountModel } from '../models/account/account' import { VideoChannelModel } from '../models/video/video-channel' import { extname, join } from 'path' import { retryTransactionWrapper } from '../helpers/database-utils' +import * as uuidv4 from 'uuid/v4' async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { const extension = extname(avatarPhysicalFile.filename) - const avatarName = accountOrChannel.Actor.uuid + extension + const avatarName = uuidv4() + extension const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) -- cgit v1.2.3 From edb4ffc7e0b13659d7c73b120f2c87b27e4c26a1 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 8 Oct 2018 09:26:04 -0500 Subject: Set bitrate limits for transcoding (fixes #638) (#1135) * Set bitrate limits for transcoding (fixes #638) * added optimization script and test, changed stuff * fix test, improve docs * re-add optimize-old-videos script * added documentation * Don't optimize videos without valid UUID, or redundancy videos * move getUUIDFromFilename * fix tests? * update torrent and file size, some more fixes/improvements * use higher bitrate for high fps video, adjust bitrates * add test video * don't throw error if resolution is undefined * generate test fixture on the fly * use random noise video for bitrate test, add promise * shorten test video to avoid timeout * use existing function to optimize video * various fixes * increase test timeout * limit test fixture size, add link * test fixes * add await * more test fixes, add -b:v parameter * replace ffmpeg wiki link * fix ffmpeg params * fix unit test * add test fixture to .gitgnore * add video transcoding fps model * add missing file --- server/lib/activitypub/crawl.ts | 2 +- server/lib/job-queue/handlers/video-file.ts | 4 ++-- server/lib/video-transcoding.ts | 15 ++++++++++----- 3 files changed, 13 insertions(+), 8 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 55912341c..db9ce3293 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts @@ -1,7 +1,7 @@ import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' import { doRequest } from '../../helpers/requests' import { logger } from '../../helpers/logger' -import Bluebird = require('bluebird') +import * as Bluebird from 'bluebird' async function crawlCollectionPage (uri: string, handler: (items: T[]) => Promise | Bluebird) { logger.info('Crawling ActivityPub data on %s.', uri) diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 1463c93fc..adc0a2a15 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -8,7 +8,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' -import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding' +import { importVideoFile, transcodeOriginalVideofile, optimizeVideofile } from '../../video-transcoding' export type VideoFilePayload = { videoUUID: string @@ -56,7 +56,7 @@ async function processVideoFile (job: Bull.Job) { await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) } else { - await optimizeOriginalVideofile(video) + await optimizeVideofile(video) await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) } diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index bf3ff78c2..04cadf74b 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -1,5 +1,5 @@ import { CONFIG } from '../initializers' -import { join, extname } from 'path' +import { join, extname, basename } from 'path' import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' import { copy, remove, rename, stat } from 'fs-extra' import { logger } from '../helpers/logger' @@ -7,11 +7,16 @@ import { VideoResolution } from '../../shared/models/videos' import { VideoFileModel } from '../models/video/video-file' import { VideoModel } from '../models/video/video' -async function optimizeOriginalVideofile (video: VideoModel) { +async function optimizeVideofile (video: VideoModel, videoInputPath?: string) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const newExtname = '.mp4' - const inputVideoFile = video.getOriginalFile() - const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) + let inputVideoFile = null + if (videoInputPath == null) { + inputVideoFile = video.getOriginalFile() + videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) + } else { + inputVideoFile = basename(videoInputPath) + } const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname) const transcodeOptions = { @@ -124,7 +129,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) { } export { - optimizeOriginalVideofile, + optimizeVideofile, transcodeOriginalVideofile, importVideoFile } -- cgit v1.2.3 From 9f1ddd249652c1e35b45db33885a00a005f9b059 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Oct 2018 16:50:56 +0200 Subject: Change a little bit optimize-old-videos logic --- server/lib/video-transcoding.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'server/lib') diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 04cadf74b..a78de61e5 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -1,5 +1,5 @@ import { CONFIG } from '../initializers' -import { join, extname, basename } from 'path' +import { extname, join } from 'path' import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' import { copy, remove, rename, stat } from 'fs-extra' import { logger } from '../helpers/logger' @@ -7,16 +7,12 @@ import { VideoResolution } from '../../shared/models/videos' import { VideoFileModel } from '../models/video/video-file' import { VideoModel } from '../models/video/video' -async function optimizeVideofile (video: VideoModel, videoInputPath?: string) { +async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const newExtname = '.mp4' - let inputVideoFile = null - if (videoInputPath == null) { - inputVideoFile = video.getOriginalFile() - videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) - } else { - inputVideoFile = basename(videoInputPath) - } + + const inputVideoFile = inputVideoFileArg ? inputVideoFileArg : video.getOriginalFile() + const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname) const transcodeOptions = { -- cgit v1.2.3 From 729bb184819ddda1d7313da0c30b3397e5689721 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 Oct 2018 08:51:58 +0200 Subject: Add more headers to broadcast/unicast --- .../job-queue/handlers/activitypub-http-broadcast.ts | 5 +++-- .../job-queue/handlers/activitypub-http-unicast.ts | 5 +++-- .../handlers/utils/activitypub-http-utils.ts | 19 ++++++++++++++++--- 3 files changed, 22 insertions(+), 7 deletions(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 03a9e12a4..abbd89b3b 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -3,7 +3,7 @@ import * as Bluebird from 'bluebird' import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' +import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' export type ActivitypubHttpBroadcastPayload = { @@ -25,7 +25,8 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { uri: '', json: body, httpSignature: httpSignatureOptions, - timeout: JOB_REQUEST_TIMEOUT + timeout: JOB_REQUEST_TIMEOUT, + headers: buildGlobalHeaders(body) } const badUrls: string[] = [] diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index c90d735f6..d36479032 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -2,7 +2,7 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' +import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { JOB_REQUEST_TIMEOUT } from '../../../initializers' export type ActivitypubHttpUnicastPayload = { @@ -25,7 +25,8 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { uri, json: body, httpSignature: httpSignatureOptions, - timeout: JOB_REQUEST_TIMEOUT + timeout: JOB_REQUEST_TIMEOUT, + headers: buildGlobalHeaders(body) } try { diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index 36092665e..d71c91a24 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts @@ -1,8 +1,11 @@ import { buildSignedActivity } from '../../../../helpers/activitypub' import { getServerActor } from '../../../../helpers/utils' import { ActorModel } from '../../../../models/activitypub/actor' +import { sha256 } from '../../../../helpers/core-utils' -async function computeBody (payload: { body: any, signatureActorId?: number }) { +type Payload = { body: any, signatureActorId?: number } + +async function computeBody (payload: Payload) { let body = payload.body if (payload.signatureActorId) { @@ -14,7 +17,7 @@ async function computeBody (payload: { body: any, signatureActorId?: number }) { return body } -async function buildSignedRequestOptions (payload: { signatureActorId?: number }) { +async function buildSignedRequestOptions (payload: Payload) { let actor: ActorModel | null if (payload.signatureActorId) { actor = await ActorModel.load(payload.signatureActorId) @@ -29,11 +32,21 @@ async function buildSignedRequestOptions (payload: { signatureActorId?: number } algorithm: 'rsa-sha256', authorizationHeaderName: 'Signature', keyId, - key: actor.privateKey + key: actor.privateKey, + headers: [ 'date', 'host', 'digest', '(request-target)' ] + } +} + +function buildGlobalHeaders (body: object) { + const digest = 'SHA-256=' + sha256(JSON.stringify(body), 'base64') + + return { + 'Digest': digest } } export { + buildGlobalHeaders, computeBody, buildSignedRequestOptions } -- cgit v1.2.3 From 7ad9b9846c44d198a736183fb186c2039f5236b5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 12 Oct 2018 15:26:04 +0200 Subject: Add ability for users to block an account/instance on server side --- server/lib/blocklist.ts | 40 ++++++++++++++++++++++++++++++++++++++++ server/lib/video-comment.ts | 6 ++---- 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 server/lib/blocklist.ts (limited to 'server/lib') diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts new file mode 100644 index 000000000..394c24537 --- /dev/null +++ b/server/lib/blocklist.ts @@ -0,0 +1,40 @@ +import { sequelizeTypescript } from '../initializers' +import { AccountBlocklistModel } from '../models/account/account-blocklist' +import { ServerBlocklistModel } from '../models/server/server-blocklist' + +function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { + return sequelizeTypescript.transaction(async t => { + return AccountBlocklistModel.create({ + accountId: byAccountId, + targetAccountId: targetAccountId + }, { transaction: t }) + }) +} + +function addServerInBlocklist (byAccountId: number, targetServerId: number) { + return sequelizeTypescript.transaction(async t => { + return ServerBlocklistModel.create({ + accountId: byAccountId, + targetServerId + }, { transaction: t }) + }) +} + +function removeAccountFromBlocklist (accountBlock: AccountBlocklistModel) { + return sequelizeTypescript.transaction(async t => { + return accountBlock.destroy({ transaction: t }) + }) +} + +function removeServerFromBlocklist (serverBlock: ServerBlocklistModel) { + return sequelizeTypescript.transaction(async t => { + return serverBlock.destroy({ transaction: t }) + }) +} + +export { + addAccountInBlocklist, + addServerInBlocklist, + removeAccountFromBlocklist, + removeServerFromBlocklist +} diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts index 70ba7c303..59bce7520 100644 --- a/server/lib/video-comment.ts +++ b/server/lib/video-comment.ts @@ -64,10 +64,8 @@ function buildFormattedCommentTree (resultList: ResultList): } const parentCommentThread = idx[childComment.inReplyToCommentId] - if (!parentCommentThread) { - const msg = `Cannot format video thread tree, parent ${childComment.inReplyToCommentId} not found for child ${childComment.id}` - throw new Error(msg) - } + // Maybe the parent comment was blocked by the admin/user + if (!parentCommentThread) continue parentCommentThread.children.push(childCommentThread) idx[childComment.id] = childCommentThread -- cgit v1.2.3 From af5767ffae41b2d5604e41ba9a7225c623dd6735 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 12 Oct 2018 17:26:40 +0200 Subject: Add user/instance block by users in the client --- server/lib/blocklist.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'server/lib') diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts index 394c24537..1633e500c 100644 --- a/server/lib/blocklist.ts +++ b/server/lib/blocklist.ts @@ -4,7 +4,7 @@ import { ServerBlocklistModel } from '../models/server/server-blocklist' function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { return sequelizeTypescript.transaction(async t => { - return AccountBlocklistModel.create({ + return AccountBlocklistModel.upsert({ accountId: byAccountId, targetAccountId: targetAccountId }, { transaction: t }) @@ -13,7 +13,7 @@ function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { function addServerInBlocklist (byAccountId: number, targetServerId: number) { return sequelizeTypescript.transaction(async t => { - return ServerBlocklistModel.create({ + return ServerBlocklistModel.upsert({ accountId: byAccountId, targetServerId }, { transaction: t }) -- cgit v1.2.3 From e27ff5da6ed7bc1f56f50f862b80fb0c7d8a6d98 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 18 Oct 2018 08:48:24 +0200 Subject: AP mimeType -> mediaType --- server/lib/activitypub/videos.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 54cea542f..3da363c0a 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -310,7 +310,8 @@ export { function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) - return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') + const urlMediaType = url.mediaType || url.mimeType + return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') } async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { @@ -468,7 +469,8 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid for (const fileUrl of fileUrls) { // Fetch associated magnet uri const magnet = videoObject.url.find(u => { - return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height + const mediaType = u.mediaType || u.mimeType + return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height }) if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) @@ -478,8 +480,9 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid throw new Error('Cannot parse magnet URI ' + magnet.href) } + const mediaType = fileUrl.mediaType || fileUrl.mimeType const attribute = { - extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], + extname: VIDEO_MIMETYPE_EXT[ mediaType ], infoHash: parsed.infoHash, resolution: fileUrl.height, size: fileUrl.size, -- cgit v1.2.3 From 41f2ebae4f970932fb62d2d8923b1f776f0b1494 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 19 Oct 2018 11:41:19 +0200 Subject: Add HTTP signature check before linked signature It's faster, and will allow us to use RSA signature 2018 (with upstream jsonld-signature module) without too much incompatibilities in the peertube federation --- server/lib/job-queue/handlers/utils/activitypub-http-utils.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index d71c91a24..fd9c74341 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts @@ -2,6 +2,7 @@ import { buildSignedActivity } from '../../../../helpers/activitypub' import { getServerActor } from '../../../../helpers/utils' import { ActorModel } from '../../../../models/activitypub/actor' import { sha256 } from '../../../../helpers/core-utils' +import { HTTP_SIGNATURE } from '../../../../initializers' type Payload = { body: any, signatureActorId?: number } @@ -29,11 +30,11 @@ async function buildSignedRequestOptions (payload: Payload) { const keyId = actor.getWebfingerUrl() return { - algorithm: 'rsa-sha256', - authorizationHeaderName: 'Signature', + algorithm: HTTP_SIGNATURE.ALGORITHM, + authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, keyId, key: actor.privateKey, - headers: [ 'date', 'host', 'digest', '(request-target)' ] + headers: HTTP_SIGNATURE.HEADERS_TO_SIGN } } -- cgit v1.2.3 From 891bc4f8bfa6ed8517dfffc9445db608a73d4917 Mon Sep 17 00:00:00 2001 From: BO41 Date: Tue, 13 Nov 2018 12:11:33 +0100 Subject: change video type --- server/lib/client-html.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/lib') diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index fc013e0c3..006b25bfd 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -116,7 +116,7 @@ export class ClientHtml { 'og:video:url': embedUrl, 'og:video:secure_url': embedUrl, - 'og:video:type': 'text/html', + 'og:video:type': 'video/mp4', 'og:video:width': EMBED_SIZE.width, 'og:video:height': EMBED_SIZE.height, -- cgit v1.2.3 From fb651cf2d426e7429a7c571ab7d93becd63ad116 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 14 Nov 2018 08:18:50 +0100 Subject: Revert change og video type --- server/lib/client-html.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/lib') diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 006b25bfd..fc013e0c3 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -116,7 +116,7 @@ export class ClientHtml { 'og:video:url': embedUrl, 'og:video:secure_url': embedUrl, - 'og:video:type': 'video/mp4', + 'og:video:type': 'text/html', 'og:video:width': EMBED_SIZE.width, 'og:video:height': EMBED_SIZE.height, -- cgit v1.2.3 From df66d81583e07ce049daeeef1edc6a87b57b3684 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 23 Oct 2018 11:38:48 +0200 Subject: Add compatibility with other Linked Signature algorithms --- .../lib/job-queue/handlers/utils/activitypub-http-utils.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index fd9c74341..4961d4502 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts @@ -38,15 +38,20 @@ async function buildSignedRequestOptions (payload: Payload) { } } -function buildGlobalHeaders (body: object) { - const digest = 'SHA-256=' + sha256(JSON.stringify(body), 'base64') - +function buildGlobalHeaders (body: any) { return { - 'Digest': digest + 'Digest': buildDigest(body) } } +function buildDigest (body: any) { + const rawBody = typeof body === 'string' ? body : JSON.stringify(body) + + return 'SHA-256=' + sha256(rawBody, 'base64') +} + export { + buildDigest, buildGlobalHeaders, computeBody, buildSignedRequestOptions -- cgit v1.2.3 From 5c6d985faeef1d6793d3f44ca6374f1a9b722806 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 14 Nov 2018 15:01:28 +0100 Subject: Check activities host --- server/lib/activitypub/actor.ts | 13 ++++++-- server/lib/activitypub/crawl.ts | 5 +-- server/lib/activitypub/process/index.ts | 8 ----- server/lib/activitypub/process/process-create.ts | 5 ++- server/lib/activitypub/process/process-like.ts | 4 ++- server/lib/activitypub/process/process-undo.ts | 6 ++-- server/lib/activitypub/process/process.ts | 25 ++++++++++----- server/lib/activitypub/send/send-create.ts | 10 +++--- server/lib/activitypub/send/send-like.ts | 2 +- server/lib/activitypub/send/send-undo.ts | 2 +- server/lib/activitypub/share.ts | 15 ++++++--- server/lib/activitypub/url.ts | 12 ++++---- server/lib/activitypub/video-comments.ts | 17 ++++++++++ server/lib/activitypub/video-rates.ts | 36 +++++++++++++++++++--- server/lib/activitypub/videos.ts | 7 ++++- .../job-queue/handlers/activitypub-http-fetcher.ts | 2 +- 16 files changed, 122 insertions(+), 47 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 45dd4443d..b16a00669 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -5,7 +5,7 @@ import * as url from 'url' import * as uuidv4 from 'uuid/v4' import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' -import { getActorUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub' import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' @@ -65,8 +65,12 @@ async function getOrCreateActorAndServerAndModel ( const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) + if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) { + throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`) + } + try { - // Assert we don't recurse another time + // Don't recurse another time ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) } catch (err) { logger.error('Cannot get or create account attributed to video channel ' + actor.url) @@ -297,12 +301,15 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe normalizeActor(requestResult.body) const actorJSON: ActivityPubActor = requestResult.body - if (isActorObjectValid(actorJSON) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON }) return { result: undefined, statusCode: requestResult.response.statusCode } } + if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { + throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id) + } + const followersCount = await fetchActorTotalItems(actorJSON.followers) const followingCount = await fetchActorTotalItems(actorJSON.following) diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index db9ce3293..1b9b14c2e 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts @@ -2,6 +2,7 @@ import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' import { doRequest } from '../../helpers/requests' import { logger } from '../../helpers/logger' import * as Bluebird from 'bluebird' +import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' async function crawlCollectionPage (uri: string, handler: (items: T[]) => Promise | Bluebird) { logger.info('Crawling ActivityPub data on %s.', uri) @@ -14,7 +15,7 @@ async function crawlCollectionPage (uri: string, handler: (items: T[]) => Pr timeout: JOB_REQUEST_TIMEOUT } - const response = await doRequest(options) + const response = await doRequest>(options) const firstBody = response.body let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT @@ -23,7 +24,7 @@ async function crawlCollectionPage (uri: string, handler: (items: T[]) => Pr while (nextLink && i < limit) { options.uri = nextLink - const { body } = await doRequest(options) + const { body } = await doRequest>(options) nextLink = body.next i++ diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts index db4980a72..5466739c1 100644 --- a/server/lib/activitypub/process/index.ts +++ b/server/lib/activitypub/process/index.ts @@ -1,9 +1 @@ export * from './process' -export * from './process-accept' -export * from './process-announce' -export * from './process-create' -export * from './process-delete' -export * from './process-follow' -export * from './process-like' -export * from './process-undo' -export * from './process-update' diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index cefe89db0..920d02cd2 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -12,6 +12,8 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' import { createOrUpdateCacheFile } from '../cache-file' +import { immutableAssign } from '../../../tests/utils' +import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { const activityObject = activity.object @@ -65,9 +67,10 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea videoId: video.id, accountId: byAccount.id } + const [ , created ] = await AccountVideoRateModel.findOrCreate({ where: rate, - defaults: rate, + defaults: immutableAssign(rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), transaction: t }) if (created === true) await video.increment('dislikes', { transaction: t }) diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index f7200db61..0dca17551 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -5,6 +5,8 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { ActorModel } from '../../../models/activitypub/actor' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { immutableAssign } from '../../../tests/utils' +import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { return retryTransactionWrapper(processLikeVideo, byActor, activity) @@ -34,7 +36,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { } const [ , created ] = await AccountVideoRateModel.findOrCreate({ where: rate, - defaults: rate, + defaults: immutableAssign(rate, { url: getVideoLikeActivityPubUrl(byActor, video) }), transaction: t }) if (created === true) await video.increment('likes', { transaction: t }) diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index ff019cd8c..438a013b6 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -55,7 +55,8 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { return sequelizeTypescript.transaction(async t => { if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) + let rate = await AccountVideoRateModel.loadByUrl(likeActivity.id, t) + if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) await rate.destroy({ transaction: t }) @@ -78,7 +79,8 @@ async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) return sequelizeTypescript.transaction(async t => { if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) + let rate = await AccountVideoRateModel.loadByUrl(dislike.id, t) + if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) await rate.destroy({ transaction: t }) diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index b263f1ea2..b9b255ddf 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -1,5 +1,5 @@ import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { getActorUrl } from '../../../helpers/activitypub' +import { checkUrlsSameHost, getActorUrl } from '../../../helpers/activitypub' import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/activitypub/actor' import { processAcceptActivity } from './process-accept' @@ -25,11 +25,17 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: Ac Like: processLikeActivity } -async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { +async function processActivities ( + activities: Activity[], + options: { + signatureActor?: ActorModel + inboxActor?: ActorModel + outboxUrl?: string + } = {}) { const actorsCache: { [ url: string ]: ActorModel } = {} for (const activity of activities) { - if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { + if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) continue } @@ -37,12 +43,17 @@ async function processActivities (activities: Activity[], signatureActor?: Actor const actorUrl = getActorUrl(activity.actor) // When we fetch remote data, we don't have signature - if (signatureActor && actorUrl !== signatureActor.url) { - logger.warn('Signature mismatch between %s and %s.', actorUrl, signatureActor.url) + if (options.signatureActor && actorUrl !== options.signatureActor.url) { + logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, options.signatureActor.url) continue } - const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) + if (options.outboxUrl && checkUrlsSameHost(options.outboxUrl, actorUrl) !== true) { + logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', options.outboxUrl, actorUrl) + continue + } + + const byActor = options.signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) actorsCache[actorUrl] = byActor const activityProcessor = processActivity[activity.type] @@ -52,7 +63,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor } try { - await activityProcessor(activity, byActor, inboxActor) + await activityProcessor(activity, byActor, options.inboxActor) } catch (err) { logger.warn('Cannot process activity %s.', activity.type, { err }) } diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 285edba3b..e3fca0a17 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -95,7 +95,7 @@ async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transa logger.info('Creating job to send view of %s.', video.url) const url = getVideoViewActivityPubUrl(byActor, video) - const viewActivity = buildViewActivity(byActor, video) + const viewActivity = buildViewActivity(url, byActor, video) return sendVideoRelatedCreateActivity({ // Use the server actor to send the view @@ -111,7 +111,7 @@ async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Tra logger.info('Creating job to dislike %s.', video.url) const url = getVideoDislikeActivityPubUrl(byActor, video) - const dislikeActivity = buildDislikeActivity(byActor, video) + const dislikeActivity = buildDislikeActivity(url, byActor, video) return sendVideoRelatedCreateActivity({ byActor, @@ -136,16 +136,18 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud ) } -function buildDislikeActivity (byActor: ActorModel, video: VideoModel) { +function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel) { return { + id: url, type: 'Dislike', actor: byActor.url, object: video.url } } -function buildViewActivity (byActor: ActorModel, video: VideoModel) { +function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel) { return { + id: url, type: 'View', actor: byActor.url, object: video.url diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts index 89307acc6..35227887a 100644 --- a/server/lib/activitypub/send/send-like.ts +++ b/server/lib/activitypub/send/send-like.ts @@ -24,8 +24,8 @@ function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, return audiencify( { - type: 'Like' as 'Like', id: url, + type: 'Like' as 'Like', actor: byActor.url, object: video.url }, diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index 5236d2cb3..bf1b6e117 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -64,7 +64,7 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans logger.info('Creating job to undo a dislike of video %s.', video.url) const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) - const dislikeActivity = buildDislikeActivity(byActor, video) + const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 3ff60a97c..d2649e2d5 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -4,13 +4,14 @@ import { getServerActor } from '../../helpers/utils' import { VideoModel } from '../../models/video/video' import { VideoShareModel } from '../../models/video/video-share' import { sendUndoAnnounce, sendVideoAnnounce } from './send' -import { getAnnounceActivityPubUrl } from './url' +import { getVideoAnnounceActivityPubUrl } from './url' import { VideoChannelModel } from '../../models/video/video-channel' import * as Bluebird from 'bluebird' import { doRequest } from '../../helpers/requests' import { getOrCreateActorAndServerAndModel } from './actor' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' +import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub' async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -38,9 +39,13 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { json: true, activityPub: true }) - if (!body || !body.actor) throw new Error('Body of body actor is invalid') + if (!body || !body.actor) throw new Error('Body or body actor is invalid') + + const actorUrl = getActorUrl(body.actor) + if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { + throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) + } - const actorUrl = body.actor const actor = await getOrCreateActorAndServerAndModel(actorUrl) const entry = { @@ -72,7 +77,7 @@ export { async function shareByServer (video: VideoModel, t: Transaction) { const serverActor = await getServerActor() - const serverShareUrl = getAnnounceActivityPubUrl(video.url, serverActor) + const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video) return VideoShareModel.findOrCreate({ defaults: { actorId: serverActor.id, @@ -91,7 +96,7 @@ async function shareByServer (video: VideoModel, t: Transaction) { } async function shareByVideoChannel (video: VideoModel, t: Transaction) { - const videoChannelShareUrl = getAnnounceActivityPubUrl(video.url, video.VideoChannel.Actor) + const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) return VideoShareModel.findOrCreate({ defaults: { actorId: video.VideoChannel.actorId, diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index e792be698..38f15448c 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts @@ -33,14 +33,14 @@ function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) { } function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { - return video.url + '/views/' + byActor.uuid + '/' + new Date().toISOString() + return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString() } -function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { +function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) { return byActor.url + '/likes/' + video.id } -function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { +function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) { return byActor.url + '/dislikes/' + video.id } @@ -74,8 +74,8 @@ function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) { return follower.url + '/accepts/follows/' + me.id } -function getAnnounceActivityPubUrl (originalUrl: string, byActor: ActorModel) { - return originalUrl + '/announces/' + byActor.id +function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) { + return video.url + '/announces/' + byActor.id } function getDeleteActivityPubUrl (originalUrl: string) { @@ -97,7 +97,7 @@ export { getVideoAbuseActivityPubUrl, getActorFollowActivityPubUrl, getActorFollowAcceptActivityPubUrl, - getAnnounceActivityPubUrl, + getVideoAnnounceActivityPubUrl, getUpdateActivityPubUrl, getUndoActivityPubUrl, getVideoViewActivityPubUrl, diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index c8c17f4c4..5868e7297 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -9,6 +9,7 @@ import { VideoCommentModel } from '../../models/video/video-comment' import { getOrCreateActorAndServerAndModel } from './actor' import { getOrCreateVideoAndAccountAndChannel } from './videos' import * as Bluebird from 'bluebird' +import { checkUrlsSameHost } from '../../helpers/activitypub' async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { let originCommentId: number = null @@ -61,6 +62,14 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { const actorUrl = body.attributedTo if (!actorUrl) return { created: false } + if (checkUrlsSameHost(commentUrl, actorUrl) !== true) { + throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${commentUrl}`) + } + + if (checkUrlsSameHost(body.id, commentUrl) !== true) { + throw new Error(`Comment url ${commentUrl} host is different from the AP object id ${body.id}`) + } + const actor = await getOrCreateActorAndServerAndModel(actorUrl) const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) if (!entry) return { created: false } @@ -134,6 +143,14 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { const actorUrl = body.attributedTo if (!actorUrl) throw new Error('Miss attributed to in comment') + if (checkUrlsSameHost(url, actorUrl) !== true) { + throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`) + } + + if (checkUrlsSameHost(body.id, url) !== true) { + throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`) + } + const actor = await getOrCreateActorAndServerAndModel(actorUrl) const comment = new VideoCommentModel({ url: body.id, diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 1619251c3..1854b44c4 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -8,13 +8,35 @@ import { getOrCreateActorAndServerAndModel } from './actor' import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' +import { doRequest } from '../../helpers/requests' +import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub' +import { ActorModel } from '../../models/activitypub/actor' +import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' -async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { +async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { let rateCounts = 0 - await Bluebird.map(actorUrls, async actorUrl => { + await Bluebird.map(ratesUrl, async rateUrl => { try { + // Fetch url + const { body } = await doRequest({ + uri: rateUrl, + json: true, + activityPub: true + }) + if (!body || !body.actor) throw new Error('Body or body actor is invalid') + + const actorUrl = getActorUrl(body.actor) + if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { + throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) + } + + if (checkUrlsSameHost(body.id, rateUrl) !== true) { + throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`) + } + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const [ , created ] = await AccountVideoRateModel .findOrCreate({ where: { @@ -24,13 +46,14 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR defaults: { videoId: video.id, accountId: actor.Account.id, - type: rate + type: rate, + url: body.id } }) if (created) rateCounts += 1 } catch (err) { - logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) + logger.warn('Cannot add rate %s.', rateUrl, { err }) } }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) @@ -62,7 +85,12 @@ async function sendVideoRateChange (account: AccountModel, if (dislikes > 0) await sendCreateDislike(actor, video, t) } +function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { + return rateType === 'like' ? getVideoLikeActivityPubUrl(actor, video) : getVideoDislikeActivityPubUrl(actor, video) +} + export { + getRateUrl, createRates, sendVideoRateChange } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 3da363c0a..5bd03c8c6 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -29,6 +29,7 @@ import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' +import { checkUrlsSameHost } from '../../helpers/activitypub' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -63,7 +64,7 @@ async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request. const { response, body } = await doRequest(options) - if (sanitizeAndCheckVideoTorrentObject(body) === false) { + if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { logger.debug('Remote video JSON is not valid.', { body }) return { response, videoObject: undefined } } @@ -107,6 +108,10 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject const channel = videoObject.attributedTo.find(a => a.type === 'Group') if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) + if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { + throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) + } + return getOrCreateActorAndServerAndModel(channel.id, 'all') } diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 42217c27c..67ccfa995 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -23,7 +23,7 @@ async function processActivityPubHttpFetcher (job: Bull.Job) { if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise } = { - 'activity': items => processActivities(items), + 'activity': items => processActivities(items, { outboxUrl: payload.uri }), 'video-likes': items => createRates(items, video, 'like'), 'video-dislikes': items => createRates(items, video, 'dislike'), 'video-shares': items => addVideoShares(items, video), -- cgit v1.2.3 From 742ddee1f131e6a2d701f2eeeb2851e8e1020cb2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 15 Nov 2018 10:07:44 +0100 Subject: Fix server redundancy tests --- server/lib/schedulers/videos-redundancy-scheduler.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'server/lib') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c49a8c89a..8b7f33539 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -185,11 +185,12 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { - const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate) + const maxSize = redundancy.size const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) + const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) - return totalDuplicated > maxSize + return totalWillDuplicate > maxSize } private buildNewExpiration (expiresAfterMs: number) { -- cgit v1.2.3 From 030177d246834fdba89be9bbaeac497589b47102 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 15 Nov 2018 16:18:12 +0100 Subject: Don't forward view, send updates instead To avoid inconsistencies in the federation, now the origin server will tell other instances what is the correct number of views --- server/lib/activitypub/process/process-create.ts | 16 ++++------------ server/lib/job-queue/handlers/video-views.ts | 8 ++++++-- server/lib/job-queue/job-queue.ts | 4 ++-- 3 files changed, 12 insertions(+), 16 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 920d02cd2..214e14546 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -13,7 +13,8 @@ import { forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' import { createOrUpdateCacheFile } from '../cache-file' import { immutableAssign } from '../../../tests/utils' -import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' +import { getVideoDislikeActivityPubUrl } from '../url' +import { VideoModel } from '../../../models/video/video' async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { const activityObject = activity.object @@ -87,19 +88,10 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const options = { - videoObject: view.object, - fetchType: 'only-video' as 'only-video' - } - const { video } = await getOrCreateVideoAndAccountAndChannel(options) + const video = await VideoModel.loadByUrl(view.object) + if (!video || video.isOwned() === false) return await Redis.Instance.addVideoView(video.id) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } } async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index cf180a11a..2ceec2342 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts @@ -3,8 +3,9 @@ import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' import { VideoViewModel } from '../../../models/video/video-views' import { isTestInstance } from '../../../helpers/core-utils' +import { federateVideoIfNeeded } from '../../activitypub' -async function processVideosViewsViews () { +async function processVideosViews () { const lastHour = new Date() // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour @@ -36,6 +37,9 @@ async function processVideosViewsViews () { views, videoId }) + + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) + await federateVideoIfNeeded(video, false) } catch (err) { logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour) } @@ -51,5 +55,5 @@ async function processVideosViewsViews () { // --------------------------------------------------------------------------- export { - processVideosViewsViews + processVideosViews } diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 0696ba43c..4cfd4d253 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -10,7 +10,7 @@ import { EmailPayload, processEmail } from './handlers/email' import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' import { processVideoImport, VideoImportPayload } from './handlers/video-import' -import { processVideosViewsViews } from './handlers/video-views' +import { processVideosViews } from './handlers/video-views' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -32,7 +32,7 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'video-file': processVideoFile, 'email': processEmail, 'video-import': processVideoImport, - 'videos-views': processVideosViewsViews + 'videos-views': processVideosViews } const jobTypes: JobType[] = [ -- cgit v1.2.3 From 6385c0cb7f773f5212a96807468988e17dba1d6d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 15 Nov 2018 16:57:59 +0100 Subject: Fix embed video id parsing --- server/lib/job-queue/handlers/video-views.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index 2ceec2342..f44c3c727 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts @@ -39,7 +39,7 @@ async function processVideosViews () { }) const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) - await federateVideoIfNeeded(video, false) + if (video.isOwned()) await federateVideoIfNeeded(video, false) } catch (err) { logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour) } -- cgit v1.2.3 From 92e07c3b5d9dbf2febedb1b5b87ec676eb6d1ac8 Mon Sep 17 00:00:00 2001 From: buoyantair Date: Fri, 16 Nov 2018 02:51:26 +0530 Subject: Fix dependency errors between modules --- server/lib/activitypub/process/process-create.ts | 2 +- server/lib/activitypub/process/process-like.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 214e14546..9a72cb899 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -12,7 +12,7 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' import { createOrUpdateCacheFile } from '../cache-file' -import { immutableAssign } from '../../../tests/utils' +import { immutableAssign } from '../../../../shared/utils' import { getVideoDislikeActivityPubUrl } from '../url' import { VideoModel } from '../../../models/video/video' diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 0dca17551..be86665e9 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -5,7 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { ActorModel } from '../../../models/activitypub/actor' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' -import { immutableAssign } from '../../../tests/utils' +import { immutableAssign } from '../../../../shared/utils' import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { -- cgit v1.2.3 From 58d515e32fe1d0133435b3a5e550c6ff24906fff Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 16 Nov 2018 16:48:17 +0100 Subject: Fix images size when downloading them --- server/lib/activitypub/actor.ts | 9 +++------ server/lib/activitypub/videos.ts | 10 +++------- server/lib/job-queue/handlers/video-import.ts | 10 +++++----- 3 files changed, 11 insertions(+), 18 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index b16a00669..218dbc6a7 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -11,9 +11,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' +import { doRequest, doRequestAndSaveToFile, downloadImage } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' +import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript } from '../../initializers' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' @@ -180,10 +180,7 @@ async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { const avatarName = uuidv4() + extension const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) - await doRequestAndSaveToFile({ - method: 'GET', - uri: actorJSON.icon.url - }, destPath) + await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE) return avatarName } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5bd03c8c6..80de92f24 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -10,8 +10,8 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validat import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' -import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' +import { doRequest, downloadImage } from '../../helpers/requests' +import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_MIMETYPE_EXT } from '../../initializers' import { ActorModel } from '../../models/activitypub/actor' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' @@ -97,11 +97,7 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) const thumbnailName = video.getThumbnailName() const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) - const options = { - method: 'GET', - uri: icon.url - } - return doRequestAndSaveToFile(options, thumbnailPath) + return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE) } function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index e3f2a276c..4de901c0c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -6,8 +6,8 @@ import { VideoImportState } from '../../../../shared/models/videos' import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' import { extname, join } from 'path' import { VideoFileModel } from '../../../models/video/video-file' -import { CONFIG, sequelizeTypescript, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' -import { doRequestAndSaveToFile } from '../../../helpers/requests' +import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' +import { doRequestAndSaveToFile, downloadImage } from '../../../helpers/requests' import { VideoState } from '../../../../shared' import { JobQueue } from '../index' import { federateVideoIfNeeded } from '../../activitypub' @@ -133,7 +133,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide videoId: videoImport.videoId } videoFile = new VideoFileModel(videoFileData) - // Import if the import fails, to clean files + // To clean files if the import fails videoImport.Video.VideoFiles = [ videoFile ] // Move file @@ -145,7 +145,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide if (options.downloadThumbnail) { if (options.thumbnailUrl) { const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) - await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath) + await downloadImage(options.thumbnailUrl, destThumbnailPath, THUMBNAILS_SIZE) } else { await videoImport.Video.createThumbnail(videoFile) } @@ -157,7 +157,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide if (options.downloadPreview) { if (options.thumbnailUrl) { const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) - await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath) + await downloadImage(options.thumbnailUrl, destPreviewPath, PREVIEWS_SIZE) } else { await videoImport.Video.createPreview(videoFile) } -- cgit v1.2.3 From a8a63227781c6815532cb7a68699b08fdb0368be Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 19 Nov 2018 11:24:31 +0100 Subject: Optimize image resizing --- server/lib/activitypub/videos.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 80de92f24..6ff9baefe 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -242,10 +242,6 @@ async function updateVideoFromAP (options: { if (options.updateViews === true) options.video.set('views', videoData.views) await options.video.save(sequelizeOptions) - // Don't block on request - generateThumbnailFromUrl(options.video, options.videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) - { const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) @@ -293,6 +289,12 @@ async function updateVideoFromAP (options: { logger.debug('Cannot update the remote video.', { err }) throw err } + + try { + await generateThumbnailFromUrl(options.video, options.videoObject.icon) + } catch (err) { + logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }) + } } export { -- cgit v1.2.3 From 361805c48b14c5402c9984485c67c45a1a3113cc Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 19 Nov 2018 14:34:01 +0100 Subject: Fix checkbox margins --- server/lib/activitypub/actor.ts | 8 ++++---- server/lib/activitypub/process/process.ts | 4 ++-- server/lib/activitypub/share.ts | 4 ++-- server/lib/activitypub/video-rates.ts | 4 ++-- server/lib/activitypub/videos.ts | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 218dbc6a7..504263c99 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -5,15 +5,15 @@ import * as url from 'url' import * as uuidv4 from 'uuid/v4' import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' -import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doRequest, doRequestAndSaveToFile, downloadImage } from '../../helpers/requests' +import { doRequest, downloadImage } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript } from '../../initializers' +import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' @@ -43,7 +43,7 @@ async function getOrCreateActorAndServerAndModel ( recurseIfNeeded = true, updateCollections = false ) { - const actorUrl = getActorUrl(activityActor) + const actorUrl = getAPUrl(activityActor) let created = false let actor = await fetchActorByUrl(actorUrl, fetchType) diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index b9b255ddf..bcc5cac7a 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -1,5 +1,5 @@ import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { checkUrlsSameHost, getActorUrl } from '../../../helpers/activitypub' +import { checkUrlsSameHost, getAPUrl } from '../../../helpers/activitypub' import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/activitypub/actor' import { processAcceptActivity } from './process-accept' @@ -40,7 +40,7 @@ async function processActivities ( continue } - const actorUrl = getActorUrl(activity.actor) + const actorUrl = getAPUrl(activity.actor) // When we fetch remote data, we don't have signature if (options.signatureActor && actorUrl !== options.signatureActor.url) { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index d2649e2d5..5dcba778c 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -11,7 +11,7 @@ import { doRequest } from '../../helpers/requests' import { getOrCreateActorAndServerAndModel } from './actor' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' -import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -41,7 +41,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') - const actorUrl = getActorUrl(body.actor) + const actorUrl = getAPUrl(body.actor) if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) } diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 1854b44c4..2cce67f0c 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' import { doRequest } from '../../helpers/requests' -import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' import { ActorModel } from '../../models/activitypub/actor' import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' @@ -26,7 +26,7 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') - const actorUrl = getActorUrl(body.actor) + const actorUrl = getAPUrl(body.actor) if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 6ff9baefe..4cecf9345 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -29,7 +29,7 @@ import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' -import { checkUrlsSameHost } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -167,7 +167,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { const refreshViews = options.refreshViews || false // Get video url - const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id + const videoUrl = getAPUrl(options.videoObject) let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) if (videoFromDatabase) { -- cgit v1.2.3 From 0b2f03d3712f438f67eccf86b67acd047284f9b4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 19 Nov 2018 15:21:09 +0100 Subject: Speedup peertube startup --- server/lib/user.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'server/lib') diff --git a/server/lib/user.ts b/server/lib/user.ts index db29469eb..acb883e23 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -17,8 +17,10 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse validate: validateUser } - const userCreated = await userToCreate.save(userOptions) - const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) + const [ userCreated, accountCreated ] = await Promise.all([ + userToCreate.save(userOptions), + createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) + ]) userCreated.Account = accountCreated let channelName = userCreated.username + '_channel' @@ -37,8 +39,13 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse return { user: userCreated, account: accountCreated, videoChannel } }) - account.Actor = await setAsyncActorKeys(account.Actor) - videoChannel.Actor = await setAsyncActorKeys(videoChannel.Actor) + const [ accountKeys, channelKeys ] = await Promise.all([ + setAsyncActorKeys(account.Actor), + setAsyncActorKeys(videoChannel.Actor) + ]) + + account.Actor = accountKeys + videoChannel.Actor = channelKeys return { user, account, videoChannel } as { user: UserModel, account: AccountModel, videoChannel: VideoChannelModel } } -- cgit v1.2.3 From d175a6f7ab9dd53e36f9f52769ac02dbfdc57e3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 19 Nov 2018 17:08:18 +0100 Subject: Cleanup tests imports --- server/lib/user.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'server/lib') diff --git a/server/lib/user.ts b/server/lib/user.ts index acb883e23..29d6d087d 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -17,10 +17,8 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse validate: validateUser } - const [ userCreated, accountCreated ] = await Promise.all([ - userToCreate.save(userOptions), - createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) - ]) + const userCreated = await userToCreate.save(userOptions) + const accountCreated = await createLocalAccountWithoutKeys(userCreated.username, userCreated.id, null, t) userCreated.Account = accountCreated let channelName = userCreated.username + '_channel' -- cgit v1.2.3 From 04b8c3fba614efc3827f583096c78b08cb668470 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 20 Nov 2018 10:05:51 +0100 Subject: Delete invalid or deleted remote videos --- server/lib/activitypub/process/process-update.ts | 1 - server/lib/activitypub/videos.ts | 113 +++++++++++---------- .../job-queue/handlers/activitypub-refresher.ts | 40 ++++++++ server/lib/job-queue/job-queue.ts | 8 +- 4 files changed, 103 insertions(+), 59 deletions(-) create mode 100644 server/lib/job-queue/handlers/activitypub-refresher.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index bd4013555..03831a00e 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -59,7 +59,6 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) videoObject, account: actor.Account, channel: channelActor.VideoChannel, - updateViews: true, overrideTo: activity.to } return updateVideoFromAP(updateOptions) diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 4cecf9345..998f90330 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -117,7 +117,7 @@ type SyncParam = { shares: boolean comments: boolean thumbnail: boolean - refreshVideo: boolean + refreshVideo?: boolean } async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) @@ -158,13 +158,11 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid async function getOrCreateVideoAndAccountAndChannel (options: { videoObject: VideoTorrentObject | string, syncParam?: SyncParam, - fetchType?: VideoFetchByUrlType, - refreshViews?: boolean + fetchType?: VideoFetchByUrlType }) { // Default params const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } const fetchType = options.fetchType || 'all' - const refreshViews = options.refreshViews || false // Get video url const videoUrl = getAPUrl(options.videoObject) @@ -174,11 +172,11 @@ async function getOrCreateVideoAndAccountAndChannel (options: { const refreshOptions = { video: videoFromDatabase, fetchedType: fetchType, - syncParam, - refreshViews + syncParam } - const p = refreshVideoIfNeeded(refreshOptions) - if (syncParam.refreshVideo === true) videoFromDatabase = await p + + if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) + else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) return { video: videoFromDatabase } } @@ -199,7 +197,6 @@ async function updateVideoFromAP (options: { videoObject: VideoTorrentObject, account: AccountModel, channel: VideoChannelModel, - updateViews: boolean, overrideTo?: string[] }) { logger.debug('Updating remote video "%s".', options.videoObject.uuid) @@ -238,8 +235,8 @@ async function updateVideoFromAP (options: { options.video.set('publishedAt', videoData.publishedAt) options.video.set('privacy', videoData.privacy) options.video.set('channelId', videoData.channelId) + options.video.set('views', videoData.views) - if (options.updateViews === true) options.video.set('views', videoData.views) await options.video.save(sequelizeOptions) { @@ -297,8 +294,58 @@ async function updateVideoFromAP (options: { } } +async function refreshVideoIfNeeded (options: { + video: VideoModel, + fetchedType: VideoFetchByUrlType, + syncParam: SyncParam +}): Promise { + if (!options.video.isOutdated()) return options.video + + // We need more attributes if the argument video was fetched with not enough joints + const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) + + try { + const { response, videoObject } = await fetchRemoteVideo(video.url) + if (response.statusCode === 404) { + logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) + + // Video does not exist anymore + await video.destroy() + return undefined + } + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video %s: invalid body.', video.url) + + await video.setAsRefreshed() + return video + } + + const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) + const account = await AccountModel.load(channelActor.VideoChannel.accountId) + + const updateOptions = { + video, + videoObject, + account, + channel: channelActor.VideoChannel + } + await retryTransactionWrapper(updateVideoFromAP, updateOptions) + await syncVideoExternalAttributes(video, videoObject, options.syncParam) + + return video + } catch (err) { + logger.warn('Cannot refresh video %s.', options.video.url, { err }) + + // Don't refresh in loop + await video.setAsRefreshed() + return video + } +} + export { updateVideoFromAP, + refreshVideoIfNeeded, federateVideoIfNeeded, fetchRemoteVideo, getOrCreateVideoAndAccountAndChannel, @@ -362,52 +409,6 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor return videoCreated } -async function refreshVideoIfNeeded (options: { - video: VideoModel, - fetchedType: VideoFetchByUrlType, - syncParam: SyncParam, - refreshViews: boolean -}): Promise { - if (!options.video.isOutdated()) return options.video - - // We need more attributes if the argument video was fetched with not enough joints - const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) - - try { - const { response, videoObject } = await fetchRemoteVideo(video.url) - if (response.statusCode === 404) { - logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) - - // Video does not exist anymore - await video.destroy() - return undefined - } - - if (videoObject === undefined) { - logger.warn('Cannot refresh remote video %s: invalid body.', video.url) - return video - } - - const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) - const account = await AccountModel.load(channelActor.VideoChannel.accountId) - - const updateOptions = { - video, - videoObject, - account, - channel: channelActor.VideoChannel, - updateViews: options.refreshViews - } - await retryTransactionWrapper(updateVideoFromAP, updateOptions) - await syncVideoExternalAttributes(video, videoObject, options.syncParam) - - return video - } catch (err) { - logger.warn('Cannot refresh video %s.', options.video.url, { err }) - return video - } -} - async function videoActivityObjectToDBAttributes ( videoChannel: VideoChannelModel, videoObject: VideoTorrentObject, diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts new file mode 100644 index 000000000..7752b3b40 --- /dev/null +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -0,0 +1,40 @@ +import * as Bull from 'bull' +import { logger } from '../../../helpers/logger' +import { fetchVideoByUrl } from '../../../helpers/video' +import { refreshVideoIfNeeded } from '../../activitypub' + +export type RefreshPayload = { + videoUrl: string + type: 'video' +} + +async function refreshAPObject (job: Bull.Job) { + const payload = job.data as RefreshPayload + logger.info('Processing AP refresher in job %d.', job.id) + + if (payload.type === 'video') return refreshAPVideo(payload.videoUrl) +} + +// --------------------------------------------------------------------------- + +export { + refreshAPObject +} + +// --------------------------------------------------------------------------- + +async function refreshAPVideo (videoUrl: string) { + const fetchType = 'all' as 'all' + const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } + + const videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) + if (videoFromDatabase) { + const refreshOptions = { + video: videoFromDatabase, + fetchedType: fetchType, + syncParam + } + + await refreshVideoIfNeeded(refreshOptions) + } +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 4cfd4d253..5862e178f 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -11,6 +11,7 @@ import { processVideoFile, processVideoFileImport, VideoFileImportPayload, Video import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' import { processVideoImport, VideoImportPayload } from './handlers/video-import' import { processVideosViews } from './handlers/video-views' +import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -21,6 +22,7 @@ type CreateJobArgument = { type: 'video-file', payload: VideoFilePayload } | { type: 'email', payload: EmailPayload } | { type: 'video-import', payload: VideoImportPayload } | + { type: 'activitypub-refresher', payload: RefreshPayload } | { type: 'videos-views', payload: {} } const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { @@ -32,7 +34,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'video-file': processVideoFile, 'email': processEmail, 'video-import': processVideoImport, - 'videos-views': processVideosViews + 'videos-views': processVideosViews, + 'activitypub-refresher': refreshAPObject } const jobTypes: JobType[] = [ @@ -44,7 +47,8 @@ const jobTypes: JobType[] = [ 'video-file', 'video-file-import', 'video-import', - 'videos-views' + 'videos-views', + 'activitypub-refresher' ] class JobQueue { -- cgit v1.2.3 From a8f378e02c1b0dbb6d6ac202a369d0df18eb9317 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 22 Nov 2018 15:30:41 +0100 Subject: Don't import test tools in core --- server/lib/activitypub/process/process-create.ts | 3 +-- server/lib/activitypub/process/process-like.ts | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 214e14546..f7fb09fba 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -12,7 +12,6 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' import { createOrUpdateCacheFile } from '../cache-file' -import { immutableAssign } from '../../../tests/utils' import { getVideoDislikeActivityPubUrl } from '../url' import { VideoModel } from '../../../models/video/video' @@ -71,7 +70,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea const [ , created ] = await AccountVideoRateModel.findOrCreate({ where: rate, - defaults: immutableAssign(rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), + defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), transaction: t }) if (created === true) await video.increment('dislikes', { transaction: t }) diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 0dca17551..e8e97eece 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -5,8 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { ActorModel } from '../../../models/activitypub/actor' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' -import { immutableAssign } from '../../../tests/utils' -import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' +import { getVideoLikeActivityPubUrl } from '../url' async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { return retryTransactionWrapper(processLikeVideo, byActor, activity) @@ -36,7 +35,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { } const [ , created ] = await AccountVideoRateModel.findOrCreate({ where: rate, - defaults: immutableAssign(rate, { url: getVideoLikeActivityPubUrl(byActor, video) }), + defaults: Object.assign({}, rate, { url: getVideoLikeActivityPubUrl(byActor, video) }), transaction: t }) if (created === true) await video.increment('likes', { transaction: t }) -- cgit v1.2.3 From dbe6aa698eaacf9125d2c4232dee6e3e1f0d7ba1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 3 Dec 2018 09:14:56 +0100 Subject: Fix trending page --- server/lib/activitypub/process/process-create.ts | 13 +++++++++++-- server/lib/job-queue/handlers/video-views.ts | 13 +++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index f7fb09fba..cd7ea01aa 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -87,10 +87,19 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const video = await VideoModel.loadByUrl(view.object) - if (!video || video.isOwned() === false) return + const options = { + videoObject: view.object, + fetchType: 'only-video' as 'only-video' + } + const { video } = await getOrCreateVideoAndAccountAndChannel(options) await Redis.Instance.addVideoView(video.id) + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + await forwardVideoRelatedActivity(activity, undefined, exceptions, video) + } } async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index f44c3c727..038ef43e2 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts @@ -24,12 +24,10 @@ async function processVideosViews () { try { const views = await Redis.Instance.getVideoViews(videoId, hour) if (isNaN(views)) { - logger.error('Cannot process videos views of video %d in hour %d: views number is NaN.', videoId, hour) + logger.error('Cannot process videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, views) } else { logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) - await VideoModel.incrementViews(videoId, views) - try { await VideoViewModel.create({ startDate, @@ -39,7 +37,14 @@ async function processVideosViews () { }) const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) - if (video.isOwned()) await federateVideoIfNeeded(video, false) + if (video.isOwned()) { + // If this is a remote video, the origin instance will send us an update + await VideoModel.incrementViews(videoId, views) + + // Send video update + video.views += views + await federateVideoIfNeeded(video, false) + } } catch (err) { logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour) } -- cgit v1.2.3 From f9a971c671d5a8b88f420a86656a788575105598 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 09:34:29 +0100 Subject: Update dependencies --- server/lib/job-queue/handlers/video-file.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index adc0a2a15..ddbf6d1c2 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -1,5 +1,5 @@ import * as Bull from 'bull' -import { VideoResolution, VideoState } from '../../../../shared' +import { VideoResolution, VideoState, Job } from '../../../../shared' import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' import { JobQueue } from '../job-queue' @@ -111,7 +111,7 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole ) if (resolutionsEnabled.length !== 0) { - const tasks: Bluebird[] = [] + const tasks: Bluebird>[] = [] for (const resolution of resolutionsEnabled) { const dataInput = { -- cgit v1.2.3 From 745778256ced65415b04a9817fc49db70d4b6681 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 15:12:54 +0100 Subject: Fix thumbnail processing --- server/lib/activitypub/process/process-update.ts | 2 +- server/lib/activitypub/videos.ts | 21 +++++++++++++-------- .../lib/job-queue/handlers/activitypub-refresher.ts | 3 ++- 3 files changed, 16 insertions(+), 10 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 03831a00e..c6b42d846 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -51,7 +51,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) return undefined } - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id, allowRefresh: false }) const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) const updateOptions = { diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 998f90330..a5d649391 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -158,25 +158,30 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid async function getOrCreateVideoAndAccountAndChannel (options: { videoObject: VideoTorrentObject | string, syncParam?: SyncParam, - fetchType?: VideoFetchByUrlType + fetchType?: VideoFetchByUrlType, + allowRefresh?: boolean // true by default }) { // Default params const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } const fetchType = options.fetchType || 'all' + const allowRefresh = options.allowRefresh !== false // Get video url const videoUrl = getAPUrl(options.videoObject) let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) if (videoFromDatabase) { - const refreshOptions = { - video: videoFromDatabase, - fetchedType: fetchType, - syncParam - } - if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) - else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) + if (allowRefresh === true) { + const refreshOptions = { + video: videoFromDatabase, + fetchedType: fetchType, + syncParam + } + + if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) + else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) + } return { video: videoFromDatabase } } diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 7752b3b40..671b0f487 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -10,7 +10,8 @@ export type RefreshPayload = { async function refreshAPObject (job: Bull.Job) { const payload = job.data as RefreshPayload - logger.info('Processing AP refresher in job %d.', job.id) + + logger.info('Processing AP refresher in job %d for video %s.', job.id, payload.videoUrl) if (payload.type === 'video') return refreshAPVideo(payload.videoUrl) } -- cgit v1.2.3 From 6040f87d143a5fa01db79867ece8197c3ce7be47 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 16:02:49 +0100 Subject: Add tmp and redundancy directories --- server/lib/activitypub/actor.ts | 4 +--- server/lib/activitypub/videos.ts | 3 +-- server/lib/job-queue/handlers/video-import.ts | 9 ++++----- server/lib/job-queue/handlers/video-views.ts | 4 +--- server/lib/redis.ts | 9 ++++++++- 5 files changed, 15 insertions(+), 14 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 504263c99..bbe48833d 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -178,9 +178,7 @@ async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] const avatarName = uuidv4() + extension - const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) - - await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE) + await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) return avatarName } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index a5d649391..3d17e6846 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -95,9 +95,8 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { const thumbnailName = video.getThumbnailName() - const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) - return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE) + return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) } function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 4de901c0c..51a0b5faf 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -7,7 +7,7 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro import { extname, join } from 'path' import { VideoFileModel } from '../../../models/video/video-file' import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' -import { doRequestAndSaveToFile, downloadImage } from '../../../helpers/requests' +import { downloadImage } from '../../../helpers/requests' import { VideoState } from '../../../../shared' import { JobQueue } from '../index' import { federateVideoIfNeeded } from '../../activitypub' @@ -109,6 +109,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide let tempVideoPath: string let videoDestFile: string let videoFile: VideoFileModel + try { // Download video from youtubeDL tempVideoPath = await downloader() @@ -144,8 +145,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Process thumbnail if (options.downloadThumbnail) { if (options.thumbnailUrl) { - const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) - await downloadImage(options.thumbnailUrl, destThumbnailPath, THUMBNAILS_SIZE) + await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE) } else { await videoImport.Video.createThumbnail(videoFile) } @@ -156,8 +156,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Process preview if (options.downloadPreview) { if (options.thumbnailUrl) { - const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) - await downloadImage(options.thumbnailUrl, destPreviewPath, PREVIEWS_SIZE) + await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE) } else { await videoImport.Video.createPreview(videoFile) } diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index 038ef43e2..fa1fd13b3 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts @@ -23,9 +23,7 @@ async function processVideosViews () { for (const videoId of videoIds) { try { const views = await Redis.Instance.getVideoViews(videoId, hour) - if (isNaN(views)) { - logger.error('Cannot process videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, views) - } else { + if (views) { logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) try { diff --git a/server/lib/redis.ts b/server/lib/redis.ts index abd75d512..3e25e6a2c 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -121,7 +121,14 @@ class Redis { const key = this.generateVideoViewKey(videoId, hour) const valueString = await this.getValue(key) - return parseInt(valueString, 10) + const valueInt = parseInt(valueString, 10) + + if (isNaN(valueInt)) { + logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString) + return undefined + } + + return valueInt } async getVideosIdViewed (hour: number) { -- cgit v1.2.3 From b9fffa297f49a84df8ffd0d7b842599bc88a8e3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 17:08:55 +0100 Subject: Create redundancy endpoint --- server/lib/schedulers/videos-redundancy-scheduler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'server/lib') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 8b7f33539..2a99a665d 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -145,13 +145,13 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) await rename(tmpPath, destPath) const createdModel = await VideoRedundancyModel.create({ expiresOn: this.buildNewExpiration(redundancy.minLifetime), url: getVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), + fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), strategy: redundancy.strategy, videoFileId: file.id, actorId: serverActor.id -- cgit v1.2.3 From 3b3b18203fe73e499bf8b49b15369710df95993e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 5 Dec 2018 15:10:45 +0100 Subject: Add error when email system is not configured and using the forgot password system --- server/lib/emailer.ts | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'server/lib') diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 9327792fb..074d4ad44 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -14,6 +14,7 @@ class Emailer { private static instance: Emailer private initialized = false private transporter: Transporter + private enabled = false private constructor () {} @@ -50,6 +51,8 @@ class Emailer { tls, auth }) + + this.enabled = true } else { if (!isTestInstance()) { logger.error('Cannot use SMTP server because of lack of configuration. PeerTube will not be able to send mails!') @@ -57,6 +60,10 @@ class Emailer { } } + isEnabled () { + return this.enabled + } + async checkConnectionOrDie () { if (!this.transporter) return -- cgit v1.2.3 From 9f8ca79284f93693c734dd4b9a27b471017fc441 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 10 Dec 2018 10:54:49 +0100 Subject: Don't quit on queue error --- server/lib/job-queue/job-queue.ts | 1 - 1 file changed, 1 deletion(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 5862e178f..e34be7dcd 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -88,7 +88,6 @@ class JobQueue { queue.on('error', err => { logger.error('Error in job queue %s.', handlerName, { err }) - process.exit(-1) }) this.queues[handlerName] = queue -- cgit v1.2.3 From 14e2014acc1362cfbb770c051a7254b156cd8efb Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Dec 2018 14:52:50 +0100 Subject: Support additional video extensions --- server/lib/activitypub/actor.ts | 7 +++---- server/lib/activitypub/videos.ts | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index bbe48833d..f7bf7c65a 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -1,5 +1,4 @@ import * as Bluebird from 'bluebird' -import { join } from 'path' import { Transaction } from 'sequelize' import * as url from 'url' import * as uuidv4 from 'uuid/v4' @@ -13,7 +12,7 @@ import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' import { doRequest, downloadImage } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' +import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' @@ -172,10 +171,10 @@ async function fetchActorTotalItems (url: string) { async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { if ( - actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && + actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && isActivityPubUrlValid(actorJSON.icon.url) ) { - const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] + const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] const avatarName = uuidv4() + extension await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 3d17e6846..379c2a0d7 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -1,7 +1,6 @@ import * as Bluebird from 'bluebird' import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' -import { join } from 'path' import * as request from 'request' import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' @@ -11,7 +10,7 @@ import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { doRequest, downloadImage } from '../../helpers/requests' -import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_MIMETYPE_EXT } from '../../initializers' +import { ACTIVITY_PUB, CONFIG, MIMETYPES, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers' import { ActorModel } from '../../models/activitypub/actor' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' @@ -362,7 +361,7 @@ export { // --------------------------------------------------------------------------- function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { - const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) + const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) const urlMediaType = url.mediaType || url.mimeType return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') @@ -490,7 +489,7 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid const mediaType = fileUrl.mediaType || fileUrl.mimeType const attribute = { - extname: VIDEO_MIMETYPE_EXT[ mediaType ], + extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ], infoHash: parsed.infoHash, resolution: fileUrl.height, size: fileUrl.size, -- cgit v1.2.3 From f481c4f9f31e897a08e818f388fecdee07f57142 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Dec 2018 15:12:38 +0100 Subject: Use move instead rename To avoid EXDEV errors --- server/lib/job-queue/handlers/video-import.ts | 4 ++-- server/lib/schedulers/videos-redundancy-scheduler.ts | 4 ++-- server/lib/video-transcoding.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 51a0b5faf..63aacff98 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -14,7 +14,7 @@ import { federateVideoIfNeeded } from '../../activitypub' import { VideoModel } from '../../../models/video/video' import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' import { getSecureTorrentName } from '../../../helpers/utils' -import { remove, rename, stat } from 'fs-extra' +import { remove, move, stat } from 'fs-extra' type VideoImportYoutubeDLPayload = { type: 'youtube-dl' @@ -139,7 +139,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Move file videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) - await rename(tempVideoPath, videoDestFile) + await move(tempVideoPath, videoDestFile) tempVideoPath = null // This path is not used anymore // Process thumbnail diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 2a99a665d..15e094d39 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -6,7 +6,7 @@ import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoFileModel } from '../../models/video/video-file' import { downloadWebTorrentVideo } from '../../helpers/webtorrent' import { join } from 'path' -import { rename } from 'fs-extra' +import { move } from 'fs-extra' import { getServerActor } from '../../helpers/utils' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' @@ -146,7 +146,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) - await rename(tmpPath, destPath) + await move(tmpPath, destPath) const createdModel = await VideoRedundancyModel.create({ expiresOn: this.buildNewExpiration(redundancy.minLifetime), diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index a78de61e5..4460f46e4 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -1,7 +1,7 @@ import { CONFIG } from '../initializers' import { extname, join } from 'path' import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' -import { copy, remove, rename, stat } from 'fs-extra' +import { copy, remove, move, stat } from 'fs-extra' import { logger } from '../helpers/logger' import { VideoResolution } from '../../shared/models/videos' import { VideoFileModel } from '../models/video/video-file' @@ -30,7 +30,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi inputVideoFile.set('extname', newExtname) const videoOutputPath = video.getVideoFilePath(inputVideoFile) - await rename(videoTranscodedPath, videoOutputPath) + await move(videoTranscodedPath, videoOutputPath) const stats = await stat(videoOutputPath) const fps = await getVideoFileFPS(videoOutputPath) -- cgit v1.2.3 From 9aac44236c84f17b14ce35e358a87389766e2743 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Dec 2018 15:49:36 +0100 Subject: Add video title/description when rendering html --- server/lib/client-html.ts | 51 ++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 18 deletions(-) (limited to 'server/lib') diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index fc013e0c3..2db3f8a34 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -18,21 +18,13 @@ export class ClientHtml { ClientHtml.htmlCache = {} } - static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { - const path = ClientHtml.getIndexPath(req, res, paramLang) - if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] - - const buffer = await readFile(path) + static async getDefaultHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { + const html = await ClientHtml.getIndexHTML(req, res, paramLang) - let html = buffer.toString() - - html = ClientHtml.addTitleTag(html) - html = ClientHtml.addDescriptionTag(html) - html = ClientHtml.addCustomCSS(html) + let customHtml = ClientHtml.addTitleTag(html) + customHtml = ClientHtml.addDescriptionTag(customHtml) - ClientHtml.htmlCache[path] = html - - return html + return customHtml } static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { @@ -55,7 +47,26 @@ export class ClientHtml { return ClientHtml.getIndexHTML(req, res) } - return ClientHtml.addOpenGraphAndOEmbedTags(html, video) + let customHtml = ClientHtml.addTitleTag(html, escapeHTML(video.name)) + customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(video.description)) + customHtml = ClientHtml.addOpenGraphAndOEmbedTags(customHtml, video) + + return customHtml + } + + private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { + const path = ClientHtml.getIndexPath(req, res, paramLang) + if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] + + const buffer = await readFile(path) + + let html = buffer.toString() + + html = ClientHtml.addCustomCSS(html) + + ClientHtml.htmlCache[path] = html + + return html } private static getIndexPath (req: express.Request, res: express.Response, paramLang?: string) { @@ -81,14 +92,18 @@ export class ClientHtml { return join(__dirname, '../../../client/dist/' + buildFileLocale(lang) + '/index.html') } - private static addTitleTag (htmlStringPage: string) { - const titleTag = '' + CONFIG.INSTANCE.NAME + '' + private static addTitleTag (htmlStringPage: string, title?: string) { + let text = title || CONFIG.INSTANCE.NAME + if (title) text += ` - ${CONFIG.INSTANCE.NAME}` + + const titleTag = `${text}` return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) } - private static addDescriptionTag (htmlStringPage: string) { - const descriptionTag = `` + private static addDescriptionTag (htmlStringPage: string, description?: string) { + const content = description || CONFIG.INSTANCE.SHORT_DESCRIPTION + const descriptionTag = `` return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) } -- cgit v1.2.3 From 56b13bd193b076d32925f0ad14b755b250b803a8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Dec 2018 11:24:34 +0100 Subject: Fix federation of some videos If we don't transcode additional resolutions, and user decided to wait transcoding before publishing the video --- server/lib/job-queue/handlers/video-file.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index ddbf6d1c2..3dca2937f 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -91,15 +91,15 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { }) } -async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boolean) { - if (video === undefined) return undefined +async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { + if (videoArg === undefined) return undefined // Outside the transaction (IO on disk) - const { videoFileResolution } = await video.getOriginalFileResolution() + const { videoFileResolution } = await videoArg.getOriginalFileResolution() return sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it - const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) + let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t) // Video does not exist anymore if (!videoDatabase) return undefined @@ -128,13 +128,13 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) } else { // No transcoding to do, it's now published - video.state = VideoState.PUBLISHED - video = await video.save({ transaction: t }) + videoDatabase.state = VideoState.PUBLISHED + videoDatabase = await videoDatabase.save({ transaction: t }) - logger.info('No transcoding jobs created for video %s (no resolutions).', video.uuid) + logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) } - return federateVideoIfNeeded(video, isNewVideo, t) + return federateVideoIfNeeded(videoDatabase, isNewVideo, t) }) } -- cgit v1.2.3 From 2f5c6b2fc6e60502c2a8df4dc9029c1d87ebe30b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Dec 2018 14:31:11 +0100 Subject: Optimize actor follow scores modifications --- server/lib/cache/actor-follow-score-cache.ts | 46 +++++++++++++++++++++ server/lib/cache/index.ts | 1 + .../handlers/activitypub-http-broadcast.ts | 3 +- .../job-queue/handlers/activitypub-http-unicast.ts | 6 +-- server/lib/job-queue/job-queue.ts | 4 +- server/lib/schedulers/abstract-scheduler.ts | 18 ++++++++- server/lib/schedulers/actor-follow-scheduler.ts | 47 ++++++++++++++++++++++ .../lib/schedulers/bad-actor-follow-scheduler.ts | 30 -------------- server/lib/schedulers/remove-old-jobs-scheduler.ts | 6 +-- server/lib/schedulers/update-videos-scheduler.ts | 15 +------ .../lib/schedulers/videos-redundancy-scheduler.ts | 9 +---- .../lib/schedulers/youtube-dl-update-scheduler.ts | 2 +- 12 files changed, 125 insertions(+), 62 deletions(-) create mode 100644 server/lib/cache/actor-follow-score-cache.ts create mode 100644 server/lib/schedulers/actor-follow-scheduler.ts delete mode 100644 server/lib/schedulers/bad-actor-follow-scheduler.ts (limited to 'server/lib') diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/cache/actor-follow-score-cache.ts new file mode 100644 index 000000000..d070bde09 --- /dev/null +++ b/server/lib/cache/actor-follow-score-cache.ts @@ -0,0 +1,46 @@ +import { ACTOR_FOLLOW_SCORE } from '../../initializers' +import { logger } from '../../helpers/logger' + +// Cache follows scores, instead of writing them too often in database +// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores +class ActorFollowScoreCache { + + private static instance: ActorFollowScoreCache + private pendingFollowsScore: { [ url: string ]: number } = {} + + private constructor () {} + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + updateActorFollowsScore (goodInboxes: string[], badInboxes: string[]) { + if (goodInboxes.length === 0 && badInboxes.length === 0) return + + logger.info('Updating %d good actor follows and %d bad actor follows scores in cache.', goodInboxes.length, badInboxes.length) + + for (const goodInbox of goodInboxes) { + if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0 + + this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS + } + + for (const badInbox of badInboxes) { + if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0 + + this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY + } + } + + getPendingFollowsScoreCopy () { + return this.pendingFollowsScore + } + + clearPendingFollowsScore () { + this.pendingFollowsScore = {} + } +} + +export { + ActorFollowScoreCache +} diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts index 54eb983fa..e921d04a7 100644 --- a/server/lib/cache/index.ts +++ b/server/lib/cache/index.ts @@ -1,2 +1,3 @@ +export * from './actor-follow-score-cache' export * from './videos-preview-cache' export * from './videos-caption-cache' diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index abbd89b3b..9493945ff 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -5,6 +5,7 @@ import { doRequest } from '../../../helpers/requests' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' +import { ActorFollowScoreCache } from '../../cache' export type ActivitypubHttpBroadcastPayload = { uris: string[] @@ -38,7 +39,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { .catch(() => badUrls.push(uri)) }, { concurrency: BROADCAST_CONCURRENCY }) - return ActorFollowModel.updateActorFollowsScore(goodUrls, badUrls, undefined) + return ActorFollowScoreCache.Instance.updateActorFollowsScore(goodUrls, badUrls) } // --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index d36479032..3973dcdc8 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -1,9 +1,9 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' -import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { JOB_REQUEST_TIMEOUT } from '../../../initializers' +import { ActorFollowScoreCache } from '../../cache' export type ActivitypubHttpUnicastPayload = { uri: string @@ -31,9 +31,9 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { try { await doRequest(options) - ActorFollowModel.updateActorFollowsScore([ uri ], [], undefined) + ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], []) } catch (err) { - ActorFollowModel.updateActorFollowsScore([], [ uri ], undefined) + ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ]) throw err } diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index e34be7dcd..ba9cbe0d9 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -165,10 +165,10 @@ class JobQueue { return total } - removeOldJobs () { + async removeOldJobs () { for (const key of Object.keys(this.queues)) { const queue = this.queues[key] - queue.clean(JOB_COMPLETED_LIFETIME, 'completed') + await queue.clean(JOB_COMPLETED_LIFETIME, 'completed') } } diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts index b9d0a4d17..86ea7aa38 100644 --- a/server/lib/schedulers/abstract-scheduler.ts +++ b/server/lib/schedulers/abstract-scheduler.ts @@ -1,8 +1,11 @@ +import { logger } from '../../helpers/logger' + export abstract class AbstractScheduler { protected abstract schedulerIntervalMs: number private interval: NodeJS.Timer + private isRunning = false enable () { if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') @@ -14,5 +17,18 @@ export abstract class AbstractScheduler { clearInterval(this.interval) } - abstract execute () + async execute () { + if (this.isRunning === true) return + this.isRunning = true + + try { + await this.internalExecute() + } catch (err) { + logger.error('Cannot execute %s scheduler.', this.constructor.name, { err }) + } finally { + this.isRunning = false + } + } + + protected abstract internalExecute (): Promise } diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts new file mode 100644 index 000000000..3967be7f8 --- /dev/null +++ b/server/lib/schedulers/actor-follow-scheduler.ts @@ -0,0 +1,47 @@ +import { isTestInstance } from '../../helpers/core-utils' +import { logger } from '../../helpers/logger' +import { ActorFollowModel } from '../../models/activitypub/actor-follow' +import { AbstractScheduler } from './abstract-scheduler' +import { SCHEDULER_INTERVALS_MS } from '../../initializers' +import { ActorFollowScoreCache } from '../cache' + +export class ActorFollowScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.actorFollowScores + + private constructor () { + super() + } + + protected async internalExecute () { + await this.processPendingScores() + + await this.removeBadActorFollows() + } + + private async processPendingScores () { + const pendingScores = ActorFollowScoreCache.Instance.getPendingFollowsScoreCopy() + + ActorFollowScoreCache.Instance.clearPendingFollowsScore() + + for (const inbox of Object.keys(pendingScores)) { + await ActorFollowModel.updateFollowScore(inbox, pendingScores[inbox]) + } + } + + private async removeBadActorFollows () { + if (!isTestInstance()) logger.info('Removing bad actor follows (scheduler).') + + try { + await ActorFollowModel.removeBadActorFollows() + } catch (err) { + logger.error('Error in bad actor follows scheduler.', { err }) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/lib/schedulers/bad-actor-follow-scheduler.ts b/server/lib/schedulers/bad-actor-follow-scheduler.ts deleted file mode 100644 index 617149aaf..000000000 --- a/server/lib/schedulers/bad-actor-follow-scheduler.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { isTestInstance } from '../../helpers/core-utils' -import { logger } from '../../helpers/logger' -import { ActorFollowModel } from '../../models/activitypub/actor-follow' -import { AbstractScheduler } from './abstract-scheduler' -import { SCHEDULER_INTERVALS_MS } from '../../initializers' - -export class BadActorFollowScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.badActorFollow - - private constructor () { - super() - } - - async execute () { - if (!isTestInstance()) logger.info('Removing bad actor follows (scheduler).') - - try { - await ActorFollowModel.removeBadActorFollows() - } catch (err) { - logger.error('Error in bad actor follows scheduler.', { err }) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/remove-old-jobs-scheduler.ts b/server/lib/schedulers/remove-old-jobs-scheduler.ts index a29a6b800..4a4341ba9 100644 --- a/server/lib/schedulers/remove-old-jobs-scheduler.ts +++ b/server/lib/schedulers/remove-old-jobs-scheduler.ts @@ -14,10 +14,10 @@ export class RemoveOldJobsScheduler extends AbstractScheduler { super() } - async execute () { - if (!isTestInstance()) logger.info('Removing old jobs (scheduler).') + protected internalExecute () { + if (!isTestInstance()) logger.info('Removing old jobs in scheduler.') - JobQueue.Instance.removeOldJobs() + return JobQueue.Instance.removeOldJobs() } static get Instance () { diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index fd2edfd17..21f071f9e 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -12,23 +12,12 @@ export class UpdateVideosScheduler extends AbstractScheduler { protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.updateVideos - private isRunning = false - private constructor () { super() } - async execute () { - if (this.isRunning === true) return - this.isRunning = true - - try { - await retryTransactionWrapper(this.updateVideos.bind(this)) - } catch (err) { - logger.error('Cannot execute update videos scheduler.', { err }) - } finally { - this.isRunning = false - } + protected async internalExecute () { + return retryTransactionWrapper(this.updateVideos.bind(this)) } private async updateVideos () { diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 15e094d39..f643ee226 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -16,7 +16,6 @@ import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' export class VideosRedundancyScheduler extends AbstractScheduler { private static instance: AbstractScheduler - private executing = false protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL @@ -24,11 +23,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { super() } - async execute () { - if (this.executing) return - - this.executing = true - + protected async internalExecute () { for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) @@ -57,8 +52,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await this.extendsLocalExpiration() await this.purgeRemoteExpired() - - this.executing = false } static get Instance () { diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index 461cd045e..aa027116d 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -12,7 +12,7 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler { super() } - execute () { + protected internalExecute () { return updateYoutubeDLBinary() } -- cgit v1.2.3 From cef534ed53e4518fe0acf581bfe880788d42fc36 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 26 Dec 2018 10:36:24 +0100 Subject: Add user notification base code --- server/lib/activitypub/process/process-announce.ts | 8 +- server/lib/activitypub/process/process-create.ts | 14 +- server/lib/activitypub/video-comments.ts | 4 +- server/lib/activitypub/videos.ts | 15 +- server/lib/client-html.ts | 4 +- server/lib/emailer.ts | 116 ++++++---- server/lib/job-queue/handlers/video-file.ts | 5 +- server/lib/job-queue/handlers/video-import.ts | 2 + server/lib/notifier.ts | 235 +++++++++++++++++++++ server/lib/oauth-model.ts | 3 +- server/lib/peertube-socket.ts | 52 +++++ server/lib/schedulers/update-videos-scheduler.ts | 5 + server/lib/user.ts | 16 ++ 13 files changed, 425 insertions(+), 54 deletions(-) create mode 100644 server/lib/notifier.ts create mode 100644 server/lib/peertube-socket.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index cc88b5423..23310b41e 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -5,6 +5,8 @@ import { ActorModel } from '../../../models/activitypub/actor' import { VideoShareModel } from '../../../models/video/video-share' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { VideoPrivacy } from '../../../../shared/models/videos' +import { Notifier } from '../../notifier' async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) { return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) @@ -21,9 +23,9 @@ export { async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) + const { video, created: videoCreated } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) - return sequelizeTypescript.transaction(async t => { + await sequelizeTypescript.transaction(async t => { // Add share entry const share = { @@ -49,4 +51,6 @@ async function processVideoShare (actorAnnouncer: ActorModel, activity: Activity return undefined }) + + if (videoCreated) Notifier.Instance.notifyOnNewVideo(video) } diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index df05ee452..2e04ee843 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -13,6 +13,7 @@ import { forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' import { createOrUpdateCacheFile } from '../cache-file' import { getVideoDislikeActivityPubUrl } from '../url' +import { Notifier } from '../../notifier' async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { const activityObject = activity.object @@ -47,7 +48,9 @@ export { async function processCreateVideo (activity: ActivityCreate) { const videoToCreateData = activity.object as VideoTorrentObject - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) + const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) + + if (created) Notifier.Instance.notifyOnNewVideo(video) return video } @@ -133,7 +136,10 @@ async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateD state: VideoAbuseState.PENDING } - await VideoAbuseModel.create(videoAbuseData, { transaction: t }) + const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) + videoAbuseInstance.Video = video + + Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) }) @@ -147,7 +153,7 @@ async function processCreateVideoComment (byActor: ActorModel, activity: Activit const { video } = await resolveThread(commentObject.inReplyTo) - const { created } = await addVideoComment(video, commentObject.id) + const { comment, created } = await addVideoComment(video, commentObject.id) if (video.isOwned() && created === true) { // Don't resend the activity to the sender @@ -155,4 +161,6 @@ async function processCreateVideoComment (byActor: ActorModel, activity: Activit await forwardVideoRelatedActivity(activity, undefined, exceptions, video) } + + if (created === true) Notifier.Instance.notifyOnNewComment(comment) } diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 5868e7297..e87301fe7 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -70,7 +70,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { throw new Error(`Comment url ${commentUrl} host is different from the AP object id ${body.id}`) } - const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const actor = await getOrCreateActorAndServerAndModel(actorUrl, 'all') const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) if (!entry) return { created: false } @@ -80,6 +80,8 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { }, defaults: entry }) + comment.Account = actor.Account + comment.Video = videoInstance return { comment, created } } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 379c2a0d7..5794988a5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -29,6 +29,7 @@ import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { Notifier } from '../notifier' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -181,7 +182,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) } - return { video: videoFromDatabase } + return { video: videoFromDatabase, created: false } } const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) @@ -192,7 +193,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { await syncVideoExternalAttributes(video, fetchedVideo, syncParam) - return { video } + return { video, created: true } } async function updateVideoFromAP (options: { @@ -213,6 +214,9 @@ async function updateVideoFromAP (options: { videoFieldsSave = options.video.toJSON() + const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE + const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED + // Check actor has the right to update the video const videoChannel = options.video.VideoChannel if (videoChannel.Account.id !== options.account.id) { @@ -277,6 +281,13 @@ async function updateVideoFromAP (options: { }) options.video.VideoCaptions = await Promise.all(videoCaptionsPromises) } + + { + // Notify our users? + if (wasPrivateVideo || wasUnlistedVideo) { + Notifier.Instance.notifyOnNewVideo(options.video) + } + } }) logger.info('Remote video with uuid %s updated', options.videoObject.uuid) diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 2db3f8a34..1875ec1fc 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -115,8 +115,8 @@ export class ClientHtml { } private static addOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) { - const previewUrl = CONFIG.WEBSERVER.URL + STATIC_PATHS.PREVIEWS + video.getPreviewName() - const videoUrl = CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + const previewUrl = CONFIG.WEBSERVER.URL + video.getPreviewStaticPath() + const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() const videoNameEscaped = escapeHTML(video.name) const videoDescriptionEscaped = escapeHTML(video.description) diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 074d4ad44..d766e655b 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -1,5 +1,4 @@ import { createTransport, Transporter } from 'nodemailer' -import { UserRight } from '../../shared/models/users' import { isTestInstance } from '../helpers/core-utils' import { bunyanLogger, logger } from '../helpers/logger' import { CONFIG } from '../initializers' @@ -8,6 +7,9 @@ import { VideoModel } from '../models/video/video' import { JobQueue } from './job-queue' import { EmailPayload } from './job-queue/handlers/email' import { readFileSync } from 'fs-extra' +import { VideoCommentModel } from '../models/video/video-comment' +import { VideoAbuseModel } from '../models/video/video-abuse' +import { VideoBlacklistModel } from '../models/video/video-blacklist' class Emailer { @@ -79,50 +81,57 @@ class Emailer { } } - addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { + addNewVideoFromSubscriberNotification (to: string[], video: VideoModel) { + const channelName = video.VideoChannel.getDisplayName() + const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() + const text = `Hi dear user,\n\n` + - `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + - `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + - `If you are not the person who initiated this request, please ignore this email.\n\n` + + `Your subscription ${channelName} just published a new video: ${video.name}` + + `\n\n` + + `You can view it on ${videoUrl} ` + + `\n\n` + `Cheers,\n` + `PeerTube.` const emailPayload: EmailPayload = { - to: [ to ], - subject: 'Reset your PeerTube password', + to, + subject: channelName + ' just published a new video', text } return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - addVerifyEmailJob (to: string, verifyEmailUrl: string) { - const text = `Welcome to PeerTube,\n\n` + - `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` + - `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + - `If you are not the person who initiated this request, please ignore this email.\n\n` + + addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) { + const accountName = comment.Account.getDisplayName() + const video = comment.Video + const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() + + const text = `Hi dear user,\n\n` + + `A new comment has been posted by ${accountName} on your video ${video.name}` + + `\n\n` + + `You can view it on ${commentUrl} ` + + `\n\n` + `Cheers,\n` + `PeerTube.` const emailPayload: EmailPayload = { - to: [ to ], - subject: 'Verify your PeerTube email', + to, + subject: 'New comment on your video ' + video.name, text } return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - async addVideoAbuseReportJob (videoId: number) { - const video = await VideoModel.load(videoId) - if (!video) throw new Error('Unknown Video id during Abuse report.') + async addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) { + const videoUrl = CONFIG.WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath() const text = `Hi,\n\n` + - `Your instance received an abuse for the following video ${video.url}\n\n` + + `${CONFIG.WEBSERVER.HOST} received an abuse for the following video ${videoUrl}\n\n` + `Cheers,\n` + `PeerTube.` - const to = await UserModel.listEmailsWithRight(UserRight.MANAGE_VIDEO_ABUSES) const emailPayload: EmailPayload = { to, subject: '[PeerTube] Received a video abuse', @@ -132,16 +141,12 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - async addVideoBlacklistReportJob (videoId: number, reason?: string) { - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) - if (!video) throw new Error('Unknown Video id during Blacklist report.') - // It's not our user - if (video.remote === true) return + async addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) { + const videoName = videoBlacklist.Video.name + const videoUrl = CONFIG.WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath() - const user = await UserModel.loadById(video.VideoChannel.Account.userId) - - const reasonString = reason ? ` for the following reason: ${reason}` : '' - const blockedString = `Your video ${video.name} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.` + const reasonString = videoBlacklist.reason ? ` for the following reason: ${videoBlacklist.reason}` : '' + const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.` const text = 'Hi,\n\n' + blockedString + @@ -149,33 +154,26 @@ class Emailer { 'Cheers,\n' + `PeerTube.` - const to = user.email const emailPayload: EmailPayload = { - to: [ to ], - subject: `[PeerTube] Video ${video.name} blacklisted`, + to, + subject: `[PeerTube] Video ${videoName} blacklisted`, text } return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - async addVideoUnblacklistReportJob (videoId: number) { - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) - if (!video) throw new Error('Unknown Video id during Blacklist report.') - // It's not our user - if (video.remote === true) return - - const user = await UserModel.loadById(video.VideoChannel.Account.userId) + async addVideoUnblacklistNotification (to: string[], video: VideoModel) { + const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() const text = 'Hi,\n\n' + - `Your video ${video.name} on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` + + `Your video ${video.name} (${videoUrl}) on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` + '\n\n' + 'Cheers,\n' + `PeerTube.` - const to = user.email const emailPayload: EmailPayload = { - to: [ to ], + to, subject: `[PeerTube] Video ${video.name} unblacklisted`, text } @@ -183,6 +181,40 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } + addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { + const text = `Hi dear user,\n\n` + + `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + + `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + + `If you are not the person who initiated this request, please ignore this email.\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to: [ to ], + subject: 'Reset your PeerTube password', + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + + addVerifyEmailJob (to: string, verifyEmailUrl: string) { + const text = `Welcome to PeerTube,\n\n` + + `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` + + `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + + `If you are not the person who initiated this request, please ignore this email.\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to: [ to ], + subject: 'Verify your PeerTube email', + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) { const reasonString = reason ? ` for the following reason: ${reason}` : '' const blockedWord = blocked ? 'blocked' : 'unblocked' @@ -205,7 +237,7 @@ class Emailer { } sendMail (to: string[], subject: string, text: string) { - if (!this.transporter) { + if (!this.enabled) { throw new Error('Cannot send mail because SMTP is not configured.') } diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 3dca2937f..959cc04fa 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -9,6 +9,7 @@ import { sequelizeTypescript } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' import { importVideoFile, transcodeOriginalVideofile, optimizeVideofile } from '../../video-transcoding' +import { Notifier } from '../../notifier' export type VideoFilePayload = { videoUUID: string @@ -86,6 +87,7 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { // If the video was not published, we consider it is a new one for other instances await federateVideoIfNeeded(videoDatabase, isNewVideo, t) + if (isNewVideo) Notifier.Instance.notifyOnNewVideo(video) return undefined }) @@ -134,7 +136,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) } - return federateVideoIfNeeded(videoDatabase, isNewVideo, t) + await federateVideoIfNeeded(videoDatabase, isNewVideo, t) + if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) }) } diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 63aacff98..82edb8d5c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -15,6 +15,7 @@ import { VideoModel } from '../../../models/video/video' import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' import { getSecureTorrentName } from '../../../helpers/utils' import { remove, move, stat } from 'fs-extra' +import { Notifier } from '../../notifier' type VideoImportYoutubeDLPayload = { type: 'youtube-dl' @@ -184,6 +185,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Now we can federate the video (reload from database, we need more attributes) const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) await federateVideoIfNeeded(videoForFederation, true, t) + Notifier.Instance.notifyOnNewVideo(videoForFederation) // Update video import object videoImport.state = VideoImportState.SUCCESS diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts new file mode 100644 index 000000000..a21b50b2d --- /dev/null +++ b/server/lib/notifier.ts @@ -0,0 +1,235 @@ +import { UserNotificationSettingValue, UserNotificationType, UserRight } from '../../shared/models/users' +import { logger } from '../helpers/logger' +import { VideoModel } from '../models/video/video' +import { Emailer } from './emailer' +import { UserNotificationModel } from '../models/account/user-notification' +import { VideoCommentModel } from '../models/video/video-comment' +import { UserModel } from '../models/account/user' +import { PeerTubeSocket } from './peertube-socket' +import { CONFIG } from '../initializers/constants' +import { VideoPrivacy, VideoState } from '../../shared/models/videos' +import { VideoAbuseModel } from '../models/video/video-abuse' +import { VideoBlacklistModel } from '../models/video/video-blacklist' +import * as Bluebird from 'bluebird' + +class Notifier { + + private static instance: Notifier + + private constructor () {} + + notifyOnNewVideo (video: VideoModel): void { + // Only notify on public and published videos + if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED) return + + this.notifySubscribersOfNewVideo(video) + .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) + } + + notifyOnNewComment (comment: VideoCommentModel): void { + this.notifyVideoOwnerOfNewComment(comment) + .catch(err => logger.error('Cannot notify of new comment %s.', comment.url, { err })) + } + + notifyOnNewVideoAbuse (videoAbuse: VideoAbuseModel): void { + this.notifyModeratorsOfNewVideoAbuse(videoAbuse) + .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err })) + } + + notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void { + this.notifyVideoOwnerOfBlacklist(videoBlacklist) + .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) + } + + notifyOnVideoUnblacklist (video: VideoModel): void { + this.notifyVideoOwnerOfUnblacklist(video) + .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', video.url, { err })) + } + + private async notifySubscribersOfNewVideo (video: VideoModel) { + // List all followers that are users + const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId) + + logger.info('Notifying %d users of new video %s.', users.length, video.url) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.newVideoFromSubscription + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION, + userId: user.id, + videoId: video.id + }) + notification.Video = video + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addNewVideoFromSubscriberNotification(emails, video) + } + + return this.notify({ users, settingGetter, notificationCreator, emailSender }) + } + + private async notifyVideoOwnerOfNewComment (comment: VideoCommentModel) { + const user = await UserModel.loadByVideoId(comment.videoId) + + // Not our user or user comments its own video + if (!user || comment.Account.userId === user.id) return + + logger.info('Notifying user %s of new comment %s.', user.username, comment.url) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.newCommentOnMyVideo + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, + userId: user.id, + commentId: comment.id + }) + notification.Comment = comment + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addNewCommentOnMyVideoNotification(emails, comment) + } + + return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) + } + + private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) { + const users = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES) + if (users.length === 0) return + + logger.info('Notifying %s user/moderators of new video abuse %s.', users.length, videoAbuse.Video.url) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.videoAbuseAsModerator + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.NEW_VIDEO_ABUSE_FOR_MODERATORS, + userId: user.id, + videoAbuseId: videoAbuse.id + }) + notification.VideoAbuse = videoAbuse + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addVideoAbuseModeratorsNotification(emails, videoAbuse) + } + + return this.notify({ users, settingGetter, notificationCreator, emailSender }) + } + + private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) { + const user = await UserModel.loadByVideoId(videoBlacklist.videoId) + if (!user) return + + logger.info('Notifying user %s that its video %s has been blacklisted.', user.username, videoBlacklist.Video.url) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.blacklistOnMyVideo + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.BLACKLIST_ON_MY_VIDEO, + userId: user.id, + videoBlacklistId: videoBlacklist.id + }) + notification.VideoBlacklist = videoBlacklist + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addVideoBlacklistNotification(emails, videoBlacklist) + } + + return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) + } + + private async notifyVideoOwnerOfUnblacklist (video: VideoModel) { + const user = await UserModel.loadByVideoId(video.id) + if (!user) return + + logger.info('Notifying user %s that its video %s has been unblacklisted.', user.username, video.url) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.blacklistOnMyVideo + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.UNBLACKLIST_ON_MY_VIDEO, + userId: user.id, + videoId: video.id + }) + notification.Video = video + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addVideoUnblacklistNotification(emails, video) + } + + return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) + } + + private async notify (options: { + users: UserModel[], + notificationCreator: (user: UserModel) => Promise, + emailSender: (emails: string[]) => Promise | Bluebird, + settingGetter: (user: UserModel) => UserNotificationSettingValue + }) { + const emails: string[] = [] + + for (const user of options.users) { + if (this.isWebNotificationEnabled(options.settingGetter(user))) { + const notification = await options.notificationCreator(user) + + PeerTubeSocket.Instance.sendNotification(user.id, notification) + } + + if (this.isEmailEnabled(user, options.settingGetter(user))) { + emails.push(user.email) + } + } + + if (emails.length !== 0) { + await options.emailSender(emails) + } + } + + private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) { + if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified !== true) return false + + return value === UserNotificationSettingValue.EMAIL || value === UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL + } + + private isWebNotificationEnabled (value: UserNotificationSettingValue) { + return value === UserNotificationSettingValue.WEB_NOTIFICATION || value === UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + Notifier +} diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 5cbe60b82..2cd2ae97c 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts @@ -1,3 +1,4 @@ +import * as Bluebird from 'bluebird' import { AccessDeniedError } from 'oauth2-server' import { logger } from '../helpers/logger' import { UserModel } from '../models/account/user' @@ -37,7 +38,7 @@ function clearCacheByToken (token: string) { function getAccessToken (bearerToken: string) { logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') - if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken] + if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) .then(tokenModel => { diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts new file mode 100644 index 000000000..eb84ecd4b --- /dev/null +++ b/server/lib/peertube-socket.ts @@ -0,0 +1,52 @@ +import * as SocketIO from 'socket.io' +import { authenticateSocket } from '../middlewares' +import { UserNotificationModel } from '../models/account/user-notification' +import { logger } from '../helpers/logger' +import { Server } from 'http' + +class PeerTubeSocket { + + private static instance: PeerTubeSocket + + private userNotificationSockets: { [ userId: number ]: SocketIO.Socket } = {} + + private constructor () {} + + init (server: Server) { + const io = SocketIO(server) + + io.of('/user-notifications') + .use(authenticateSocket) + .on('connection', socket => { + const userId = socket.handshake.query.user.id + + logger.debug('User %d connected on the notification system.', userId) + + this.userNotificationSockets[userId] = socket + + socket.on('disconnect', () => { + logger.debug('User %d disconnected from SocketIO notifications.', userId) + + delete this.userNotificationSockets[userId] + }) + }) + } + + sendNotification (userId: number, notification: UserNotificationModel) { + const socket = this.userNotificationSockets[userId] + + if (!socket) return + + socket.emit('new-notification', notification.toFormattedJSON()) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + PeerTubeSocket +} diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 21f071f9e..b7fb029f1 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -5,6 +5,7 @@ import { retryTransactionWrapper } from '../../helpers/database-utils' import { federateVideoIfNeeded } from '../activitypub' import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers' import { VideoPrivacy } from '../../../shared/models/videos' +import { Notifier } from '../notifier' export class UpdateVideosScheduler extends AbstractScheduler { @@ -39,6 +40,10 @@ export class UpdateVideosScheduler extends AbstractScheduler { await video.save({ transaction: t }) await federateVideoIfNeeded(video, isNewVideo, t) + + if (oldPrivacy === VideoPrivacy.UNLISTED || oldPrivacy === VideoPrivacy.PRIVATE) { + Notifier.Instance.notifyOnNewVideo(video) + } } await schedule.destroy({ transaction: t }) diff --git a/server/lib/user.ts b/server/lib/user.ts index 29d6d087d..72127819c 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -9,6 +9,8 @@ import { createVideoChannel } from './video-channel' import { VideoChannelModel } from '../models/video/video-channel' import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' import { ActorModel } from '../models/activitypub/actor' +import { UserNotificationSettingModel } from '../models/account/user-notification-setting' +import { UserNotificationSettingValue } from '../../shared/models/users' async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) { const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { @@ -18,6 +20,8 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse } const userCreated = await userToCreate.save(userOptions) + userCreated.NotificationSetting = await createDefaultUserNotificationSettings(userCreated, t) + const accountCreated = await createLocalAccountWithoutKeys(userCreated.username, userCreated.id, null, t) userCreated.Account = accountCreated @@ -88,3 +92,15 @@ export { createUserAccountAndChannel, createLocalAccountWithoutKeys } + +// --------------------------------------------------------------------------- + +function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Transaction | undefined) { + return UserNotificationSettingModel.create({ + userId: user.id, + newVideoFromSubscription: UserNotificationSettingValue.WEB_NOTIFICATION, + newCommentOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION, + videoAbuseAsModerator: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL, + blacklistOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL + }, { transaction: t }) +} -- cgit v1.2.3 From e8d246d5267ea8b6b3114d4bcf4f34fe5f3a5241 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 28 Dec 2018 13:47:17 +0100 Subject: Add notification settings migration --- server/lib/activitypub/videos.ts | 22 +++++++++------------- server/lib/job-queue/handlers/video-file.ts | 18 +++++++++++------- server/lib/job-queue/handlers/video-import.ts | 7 ++++--- 3 files changed, 24 insertions(+), 23 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5794988a5..893768769 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -204,19 +204,17 @@ async function updateVideoFromAP (options: { overrideTo?: string[] }) { logger.debug('Updating remote video "%s".', options.videoObject.uuid) + let videoFieldsSave: any + const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE + const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED try { await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } + const sequelizeOptions = { transaction: t } videoFieldsSave = options.video.toJSON() - const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE - const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED - // Check actor has the right to update the video const videoChannel = options.video.VideoChannel if (videoChannel.Account.id !== options.account.id) { @@ -281,15 +279,13 @@ async function updateVideoFromAP (options: { }) options.video.VideoCaptions = await Promise.all(videoCaptionsPromises) } - - { - // Notify our users? - if (wasPrivateVideo || wasUnlistedVideo) { - Notifier.Instance.notifyOnNewVideo(options.video) - } - } }) + // Notify our users? + if (wasPrivateVideo || wasUnlistedVideo) { + Notifier.Instance.notifyOnNewVideo(options.video) + } + logger.info('Remote video with uuid %s updated', options.videoObject.uuid) } catch (err) { if (options.video !== undefined && videoFieldsSave !== undefined) { diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 959cc04fa..480d324dc 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -1,5 +1,5 @@ import * as Bull from 'bull' -import { VideoResolution, VideoState, Job } from '../../../../shared' +import { VideoResolution, VideoState } from '../../../../shared' import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' import { JobQueue } from '../job-queue' @@ -8,7 +8,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' -import { importVideoFile, transcodeOriginalVideofile, optimizeVideofile } from '../../video-transcoding' +import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' import { Notifier } from '../../notifier' export type VideoFilePayload = { @@ -68,7 +68,7 @@ async function processVideoFile (job: Bull.Job) { async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { if (video === undefined) return undefined - return sequelizeTypescript.transaction(async t => { + const { videoDatabase, isNewVideo } = await sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) // Video does not exist anymore @@ -87,10 +87,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { // If the video was not published, we consider it is a new one for other instances await federateVideoIfNeeded(videoDatabase, isNewVideo, t) - if (isNewVideo) Notifier.Instance.notifyOnNewVideo(video) - return undefined + return { videoDatabase, isNewVideo } }) + + if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) } async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { @@ -99,7 +100,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo // Outside the transaction (IO on disk) const { videoFileResolution } = await videoArg.getOriginalFileResolution() - return sequelizeTypescript.transaction(async t => { + const videoDatabase = await sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t) // Video does not exist anymore @@ -137,8 +138,11 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo } await federateVideoIfNeeded(videoDatabase, isNewVideo, t) - if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + + return videoDatabase }) + + if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) } // --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 82edb8d5c..29cd1198c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -180,12 +180,11 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Update video DB object video.duration = duration video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED - const videoUpdated = await video.save({ transaction: t }) + await video.save({ transaction: t }) // Now we can federate the video (reload from database, we need more attributes) const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) await federateVideoIfNeeded(videoForFederation, true, t) - Notifier.Instance.notifyOnNewVideo(videoForFederation) // Update video import object videoImport.state = VideoImportState.SUCCESS @@ -193,10 +192,12 @@ async function processFile (downloader: () => Promise, videoImport: Vide logger.info('Video %s imported.', video.uuid) - videoImportUpdated.Video = videoUpdated + videoImportUpdated.Video = videoForFederation return videoImportUpdated }) + Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video) + // Create transcoding jobs? if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { // Put uuid because we don't have id auto incremented for now -- cgit v1.2.3 From dc13348070d808d0ba3feb56a435b835c2e7e791 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 2 Jan 2019 16:37:43 +0100 Subject: Add import finished and video published notifs --- server/lib/emailer.ts | 61 +++++++++++++++++++ server/lib/job-queue/handlers/video-file.ts | 24 +++++--- server/lib/job-queue/handlers/video-import.ts | 3 + server/lib/notifier.ts | 76 ++++++++++++++++++++++++ server/lib/schedulers/update-videos-scheduler.ts | 14 ++++- server/lib/user.ts | 2 + 6 files changed, 170 insertions(+), 10 deletions(-) (limited to 'server/lib') diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index d766e655b..6dc8f2adf 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -10,6 +10,7 @@ import { readFileSync } from 'fs-extra' import { VideoCommentModel } from '../models/video/video-comment' import { VideoAbuseModel } from '../models/video/video-abuse' import { VideoBlacklistModel } from '../models/video/video-blacklist' +import { VideoImportModel } from '../models/video/video-import' class Emailer { @@ -102,6 +103,66 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } + myVideoPublishedNotification (to: string[], video: VideoModel) { + const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() + + const text = `Hi dear user,\n\n` + + `Your video ${video.name} has been published.` + + `\n\n` + + `You can view it on ${videoUrl} ` + + `\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to, + subject: `Your video ${video.name} is published`, + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + + myVideoImportSuccessNotification (to: string[], videoImport: VideoImportModel) { + const videoUrl = CONFIG.WEBSERVER.URL + videoImport.Video.getWatchStaticPath() + + const text = `Hi dear user,\n\n` + + `Your video import ${videoImport.getTargetIdentifier()} is finished.` + + `\n\n` + + `You can view the imported video on ${videoUrl} ` + + `\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to, + subject: `Your video import ${videoImport.getTargetIdentifier()} is finished`, + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + + myVideoImportErrorNotification (to: string[], videoImport: VideoImportModel) { + const importUrl = CONFIG.WEBSERVER.URL + '/my-account/video-imports' + + const text = `Hi dear user,\n\n` + + `Your video import ${videoImport.getTargetIdentifier()} encountered an error.` + + `\n\n` + + `See your videos import dashboard for more information: ${importUrl}` + + `\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to, + subject: `Your video import ${videoImport.getTargetIdentifier()} encountered an error`, + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) { const accountName = comment.Account.getDisplayName() const video = comment.Video diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 480d324dc..593e43cc5 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -68,17 +68,17 @@ async function processVideoFile (job: Bull.Job) { async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { if (video === undefined) return undefined - const { videoDatabase, isNewVideo } = await sequelizeTypescript.transaction(async t => { + const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) // Video does not exist anymore if (!videoDatabase) return undefined - let isNewVideo = false + let videoPublished = false // We transcoded the video file in another format, now we can publish it if (videoDatabase.state !== VideoState.PUBLISHED) { - isNewVideo = true + videoPublished = true videoDatabase.state = VideoState.PUBLISHED videoDatabase.publishedAt = new Date() @@ -86,12 +86,15 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { } // If the video was not published, we consider it is a new one for other instances - await federateVideoIfNeeded(videoDatabase, isNewVideo, t) + await federateVideoIfNeeded(videoDatabase, videoPublished, t) - return { videoDatabase, isNewVideo } + return { videoDatabase, videoPublished } }) - if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + if (videoPublished) { + Notifier.Instance.notifyOnNewVideo(videoDatabase) + Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) + } } async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { @@ -100,7 +103,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo // Outside the transaction (IO on disk) const { videoFileResolution } = await videoArg.getOriginalFileResolution() - const videoDatabase = await sequelizeTypescript.transaction(async t => { + const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t) // Video does not exist anymore @@ -113,6 +116,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo { resolutions: resolutionsEnabled } ) + let videoPublished = false + if (resolutionsEnabled.length !== 0) { const tasks: Bluebird>[] = [] @@ -130,6 +135,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) } else { + videoPublished = true + // No transcoding to do, it's now published videoDatabase.state = VideoState.PUBLISHED videoDatabase = await videoDatabase.save({ transaction: t }) @@ -139,10 +146,11 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo await federateVideoIfNeeded(videoDatabase, isNewVideo, t) - return videoDatabase + return { videoDatabase, videoPublished } }) if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } // --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 29cd1198c..12004dcd7 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -197,6 +197,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide }) Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video) + Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true) // Create transcoding jobs? if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { @@ -220,6 +221,8 @@ async function processFile (downloader: () => Promise, videoImport: Vide videoImport.state = VideoImportState.FAILED await videoImport.save() + Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false) + throw err } } diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts index a21b50b2d..11b0937e9 100644 --- a/server/lib/notifier.ts +++ b/server/lib/notifier.ts @@ -11,6 +11,8 @@ import { VideoPrivacy, VideoState } from '../../shared/models/videos' import { VideoAbuseModel } from '../models/video/video-abuse' import { VideoBlacklistModel } from '../models/video/video-blacklist' import * as Bluebird from 'bluebird' +import { VideoImportModel } from '../models/video/video-import' +import { AccountBlocklistModel } from '../models/account/account-blocklist' class Notifier { @@ -26,6 +28,14 @@ class Notifier { .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) } + notifyOnPendingVideoPublished (video: VideoModel): void { + // Only notify on public videos that has been published while the user waited transcoding/scheduled update + if (video.waitTranscoding === false && !video.ScheduleVideoUpdate) return + + this.notifyOwnedVideoHasBeenPublished(video) + .catch(err => logger.error('Cannot notify owner that its video %s has been published.', video.url, { err })) + } + notifyOnNewComment (comment: VideoCommentModel): void { this.notifyVideoOwnerOfNewComment(comment) .catch(err => logger.error('Cannot notify of new comment %s.', comment.url, { err })) @@ -46,6 +56,11 @@ class Notifier { .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', video.url, { err })) } + notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void { + this.notifyOwnerVideoImportIsFinished(videoImport, success) + .catch(err => logger.error('Cannot notify owner that its video import %s is finished.', videoImport.getTargetIdentifier(), { err })) + } + private async notifySubscribersOfNewVideo (video: VideoModel) { // List all followers that are users const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId) @@ -80,6 +95,9 @@ class Notifier { // Not our user or user comments its own video if (!user || comment.Account.userId === user.id) return + const accountMuted = await AccountBlocklistModel.isAccountMutedBy(user.Account.id, comment.accountId) + if (accountMuted) return + logger.info('Notifying user %s of new comment %s.', user.username, comment.url) function settingGetter (user: UserModel) { @@ -188,6 +206,64 @@ class Notifier { return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) } + private async notifyOwnedVideoHasBeenPublished (video: VideoModel) { + const user = await UserModel.loadByVideoId(video.id) + if (!user) return + + logger.info('Notifying user %s of the publication of its video %s.', user.username, video.url) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.myVideoPublished + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.MY_VIDEO_PUBLISHED, + userId: user.id, + videoId: video.id + }) + notification.Video = video + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.myVideoPublishedNotification(emails, video) + } + + return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) + } + + private async notifyOwnerVideoImportIsFinished (videoImport: VideoImportModel, success: boolean) { + const user = await UserModel.loadByVideoImportId(videoImport.id) + if (!user) return + + logger.info('Notifying user %s its video import %s is finished.', user.username, videoImport.getTargetIdentifier()) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.myVideoImportFinished + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: success ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS : UserNotificationType.MY_VIDEO_IMPORT_ERROR, + userId: user.id, + videoImportId: videoImport.id + }) + notification.VideoImport = videoImport + + return notification + } + + function emailSender (emails: string[]) { + return success + ? Emailer.Instance.myVideoImportSuccessNotification(emails, videoImport) + : Emailer.Instance.myVideoImportErrorNotification(emails, videoImport) + } + + return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) + } + private async notify (options: { users: UserModel[], notificationCreator: (user: UserModel) => Promise, diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index b7fb029f1..2618a5857 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -6,6 +6,7 @@ import { federateVideoIfNeeded } from '../activitypub' import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers' import { VideoPrivacy } from '../../../shared/models/videos' import { Notifier } from '../notifier' +import { VideoModel } from '../../models/video/video' export class UpdateVideosScheduler extends AbstractScheduler { @@ -24,8 +25,9 @@ export class UpdateVideosScheduler extends AbstractScheduler { private async updateVideos () { if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined - return sequelizeTypescript.transaction(async t => { + const publishedVideos = await sequelizeTypescript.transaction(async t => { const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) + const publishedVideos: VideoModel[] = [] for (const schedule of schedules) { const video = schedule.Video @@ -42,13 +44,21 @@ export class UpdateVideosScheduler extends AbstractScheduler { await federateVideoIfNeeded(video, isNewVideo, t) if (oldPrivacy === VideoPrivacy.UNLISTED || oldPrivacy === VideoPrivacy.PRIVATE) { - Notifier.Instance.notifyOnNewVideo(video) + video.ScheduleVideoUpdate = schedule + publishedVideos.push(video) } } await schedule.destroy({ transaction: t }) } + + return publishedVideos }) + + for (const v of publishedVideos) { + Notifier.Instance.notifyOnNewVideo(v) + Notifier.Instance.notifyOnPendingVideoPublished(v) + } } static get Instance () { diff --git a/server/lib/user.ts b/server/lib/user.ts index 72127819c..481571828 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -100,6 +100,8 @@ function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Tr userId: user.id, newVideoFromSubscription: UserNotificationSettingValue.WEB_NOTIFICATION, newCommentOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION, + myVideoImportFinished: UserNotificationSettingValue.WEB_NOTIFICATION, + myVideoPublished: UserNotificationSettingValue.WEB_NOTIFICATION, videoAbuseAsModerator: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL, blacklistOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL }, { transaction: t }) -- cgit v1.2.3 From f7cc67b455a12ccae9b0ea16876d166720364357 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 4 Jan 2019 08:56:20 +0100 Subject: Add new follow, mention and user registered notifs --- server/lib/activitypub/process/process-accept.ts | 2 + server/lib/activitypub/process/process-follow.ts | 11 +- server/lib/emailer.ts | 63 ++++++++- .../lib/job-queue/handlers/activitypub-follow.ts | 9 +- server/lib/notifier.ts | 154 ++++++++++++++++++++- server/lib/user.ts | 13 +- 6 files changed, 235 insertions(+), 17 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 89bda9c32..605705ad3 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts @@ -2,6 +2,7 @@ import { ActivityAccept } from '../../../../shared/models/activitypub' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { addFetchOutboxJob } from '../actor' +import { Notifier } from '../../notifier' async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) { if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') @@ -24,6 +25,7 @@ async function processAccept (actor: ActorModel, targetActor: ActorModel) { if (follow.state !== 'accepted') { follow.set('state', 'accepted') await follow.save() + await addFetchOutboxJob(targetActor) } } diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index 24c9085f7..a67892440 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts @@ -5,6 +5,7 @@ import { sequelizeTypescript } from '../../../initializers' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { sendAccept } from '../send' +import { Notifier } from '../../notifier' async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { const activityObject = activity.object @@ -21,13 +22,13 @@ export { // --------------------------------------------------------------------------- async function processFollow (actor: ActorModel, targetActorURL: string) { - await sequelizeTypescript.transaction(async t => { + const { actorFollow, created } = await sequelizeTypescript.transaction(async t => { const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) if (!targetActor) throw new Error('Unknown actor') if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') - const [ actorFollow ] = await ActorFollowModel.findOrCreate({ + const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({ where: { actorId: actor.id, targetActorId: targetActor.id @@ -52,8 +53,12 @@ async function processFollow (actor: ActorModel, targetActorURL: string) { actorFollow.ActorFollowing = targetActor // Target sends to actor he accepted the follow request - return sendAccept(actorFollow) + await sendAccept(actorFollow) + + return { actorFollow, created } }) + if (created) Notifier.Instance.notifyOfNewFollow(actorFollow) + logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) } diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 6dc8f2adf..3429498e7 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -11,6 +11,7 @@ import { VideoCommentModel } from '../models/video/video-comment' import { VideoAbuseModel } from '../models/video/video-abuse' import { VideoBlacklistModel } from '../models/video/video-blacklist' import { VideoImportModel } from '../models/video/video-import' +import { ActorFollowModel } from '../models/activitypub/actor-follow' class Emailer { @@ -103,6 +104,25 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } + addNewFollowNotification (to: string[], actorFollow: ActorFollowModel, followType: 'account' | 'channel') { + const followerName = actorFollow.ActorFollower.Account.getDisplayName() + const followingName = (actorFollow.ActorFollowing.VideoChannel || actorFollow.ActorFollowing.Account).getDisplayName() + + const text = `Hi dear user,\n\n` + + `Your ${followType} ${followingName} has a new subscriber: ${followerName}` + + `\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to, + subject: 'New follower on your channel ' + followingName, + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + myVideoPublishedNotification (to: string[], video: VideoModel) { const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() @@ -185,7 +205,29 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - async addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) { + addNewCommentMentionNotification (to: string[], comment: VideoCommentModel) { + const accountName = comment.Account.getDisplayName() + const video = comment.Video + const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() + + const text = `Hi dear user,\n\n` + + `${accountName} mentioned you on video ${video.name}` + + `\n\n` + + `You can view the comment on ${commentUrl} ` + + `\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to, + subject: 'Mention on video ' + video.name, + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + + addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) { const videoUrl = CONFIG.WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath() const text = `Hi,\n\n` + @@ -202,7 +244,22 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - async addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) { + addNewUserRegistrationNotification (to: string[], user: UserModel) { + const text = `Hi,\n\n` + + `User ${user.username} just registered on ${CONFIG.WEBSERVER.HOST} PeerTube instance.\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to, + subject: '[PeerTube] New user registration on ' + CONFIG.WEBSERVER.HOST, + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + + addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) { const videoName = videoBlacklist.Video.name const videoUrl = CONFIG.WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath() @@ -224,7 +281,7 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - async addVideoUnblacklistNotification (to: string[], video: VideoModel) { + addVideoUnblacklistNotification (to: string[], video: VideoModel) { const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() const text = 'Hi,\n\n' + diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts index 36d0f237b..b4d381062 100644 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/lib/job-queue/handlers/activitypub-follow.ts @@ -8,6 +8,7 @@ import { getOrCreateActorAndServerAndModel } from '../../activitypub/actor' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { ActorModel } from '../../../models/activitypub/actor' +import { Notifier } from '../../notifier' export type ActivitypubFollowPayload = { followerActorId: number @@ -42,7 +43,7 @@ export { // --------------------------------------------------------------------------- -function follow (fromActor: ActorModel, targetActor: ActorModel) { +async function follow (fromActor: ActorModel, targetActor: ActorModel) { if (fromActor.id === targetActor.id) { throw new Error('Follower is the same than target actor.') } @@ -50,7 +51,7 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) { // Same server, direct accept const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' - return sequelizeTypescript.transaction(async t => { + const actorFollow = await sequelizeTypescript.transaction(async t => { const [ actorFollow ] = await ActorFollowModel.findOrCreate({ where: { actorId: fromActor.id, @@ -68,5 +69,9 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) { // Send a notification to remote server if our follow is not already accepted if (actorFollow.state !== 'accepted') await sendFollow(actorFollow) + + return actorFollow }) + + if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewFollow(actorFollow) } diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts index 11b0937e9..2c51d7101 100644 --- a/server/lib/notifier.ts +++ b/server/lib/notifier.ts @@ -13,6 +13,8 @@ import { VideoBlacklistModel } from '../models/video/video-blacklist' import * as Bluebird from 'bluebird' import { VideoImportModel } from '../models/video/video-import' import { AccountBlocklistModel } from '../models/account/account-blocklist' +import { ActorFollowModel } from '../models/activitypub/actor-follow' +import { AccountModel } from '../models/account/account' class Notifier { @@ -38,7 +40,10 @@ class Notifier { notifyOnNewComment (comment: VideoCommentModel): void { this.notifyVideoOwnerOfNewComment(comment) - .catch(err => logger.error('Cannot notify of new comment %s.', comment.url, { err })) + .catch(err => logger.error('Cannot notify video owner of new comment %s.', comment.url, { err })) + + this.notifyOfCommentMention(comment) + .catch(err => logger.error('Cannot notify mentions of comment %s.', comment.url, { err })) } notifyOnNewVideoAbuse (videoAbuse: VideoAbuseModel): void { @@ -61,6 +66,23 @@ class Notifier { .catch(err => logger.error('Cannot notify owner that its video import %s is finished.', videoImport.getTargetIdentifier(), { err })) } + notifyOnNewUserRegistration (user: UserModel): void { + this.notifyModeratorsOfNewUserRegistration(user) + .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) + } + + notifyOfNewFollow (actorFollow: ActorFollowModel): void { + this.notifyUserOfNewActorFollow(actorFollow) + .catch(err => { + logger.error( + 'Cannot notify owner of channel %s of a new follow by %s.', + actorFollow.ActorFollowing.VideoChannel.getDisplayName(), + actorFollow.ActorFollower.Account.getDisplayName(), + err + ) + }) + } + private async notifySubscribersOfNewVideo (video: VideoModel) { // List all followers that are users const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId) @@ -90,6 +112,8 @@ class Notifier { } private async notifyVideoOwnerOfNewComment (comment: VideoCommentModel) { + if (comment.Video.isOwned() === false) return + const user = await UserModel.loadByVideoId(comment.videoId) // Not our user or user comments its own video @@ -122,11 +146,100 @@ class Notifier { return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) } - private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) { - const users = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES) + private async notifyOfCommentMention (comment: VideoCommentModel) { + const usernames = comment.extractMentions() + let users = await UserModel.listByUsernames(usernames) + + if (comment.Video.isOwned()) { + const userException = await UserModel.loadByVideoId(comment.videoId) + users = users.filter(u => u.id !== userException.id) + } + + // Don't notify if I mentioned myself + users = users.filter(u => u.Account.id !== comment.accountId) + if (users.length === 0) return - logger.info('Notifying %s user/moderators of new video abuse %s.', users.length, videoAbuse.Video.url) + const accountMutedHash = await AccountBlocklistModel.isAccountMutedByMulti(users.map(u => u.Account.id), comment.accountId) + + logger.info('Notifying %d users of new comment %s.', users.length, comment.url) + + function settingGetter (user: UserModel) { + if (accountMutedHash[user.Account.id] === true) return UserNotificationSettingValue.NONE + + return user.NotificationSetting.commentMention + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.COMMENT_MENTION, + userId: user.id, + commentId: comment.id + }) + notification.Comment = comment + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addNewCommentMentionNotification(emails, comment) + } + + return this.notify({ users, settingGetter, notificationCreator, emailSender }) + } + + private async notifyUserOfNewActorFollow (actorFollow: ActorFollowModel) { + if (actorFollow.ActorFollowing.isOwned() === false) return + + // Account follows one of our account? + let followType: 'account' | 'channel' = 'channel' + let user = await UserModel.loadByChannelActorId(actorFollow.ActorFollowing.id) + + // Account follows one of our channel? + if (!user) { + user = await UserModel.loadByAccountActorId(actorFollow.ActorFollowing.id) + followType = 'account' + } + + if (!user) return + + if (!actorFollow.ActorFollower.Account || !actorFollow.ActorFollower.Account.name) { + actorFollow.ActorFollower.Account = await actorFollow.ActorFollower.$get('Account') as AccountModel + } + const followerAccount = actorFollow.ActorFollower.Account + + const accountMuted = await AccountBlocklistModel.isAccountMutedBy(user.Account.id, followerAccount.id) + if (accountMuted) return + + logger.info('Notifying user %s of new follower: %s.', user.username, followerAccount.getDisplayName()) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.newFollow + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.NEW_FOLLOW, + userId: user.id, + actorFollowId: actorFollow.id + }) + notification.ActorFollow = actorFollow + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addNewFollowNotification(emails, actorFollow, followType) + } + + return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) + } + + private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) { + const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES) + if (moderators.length === 0) return + + logger.info('Notifying %s user/moderators of new video abuse %s.', moderators.length, videoAbuse.Video.url) function settingGetter (user: UserModel) { return user.NotificationSetting.videoAbuseAsModerator @@ -147,7 +260,7 @@ class Notifier { return Emailer.Instance.addVideoAbuseModeratorsNotification(emails, videoAbuse) } - return this.notify({ users, settingGetter, notificationCreator, emailSender }) + return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) } private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) { @@ -264,6 +377,37 @@ class Notifier { return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) } + private async notifyModeratorsOfNewUserRegistration (registeredUser: UserModel) { + const moderators = await UserModel.listWithRight(UserRight.MANAGE_USERS) + if (moderators.length === 0) return + + logger.info( + 'Notifying %s moderators of new user registration of %s.', + moderators.length, registeredUser.Account.Actor.preferredUsername + ) + + function settingGetter (user: UserModel) { + return user.NotificationSetting.newUserRegistration + } + + async function notificationCreator (user: UserModel) { + const notification = await UserNotificationModel.create({ + type: UserNotificationType.NEW_USER_REGISTRATION, + userId: user.id, + accountId: registeredUser.Account.id + }) + notification.Account = registeredUser.Account + + return notification + } + + function emailSender (emails: string[]) { + return Emailer.Instance.addNewUserRegistrationNotification(emails, registeredUser) + } + + return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) + } + private async notify (options: { users: UserModel[], notificationCreator: (user: UserModel) => Promise, diff --git a/server/lib/user.ts b/server/lib/user.ts index 481571828..9e24e85a0 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -10,7 +10,7 @@ import { VideoChannelModel } from '../models/video/video-channel' import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' import { ActorModel } from '../models/activitypub/actor' import { UserNotificationSettingModel } from '../models/account/user-notification-setting' -import { UserNotificationSettingValue } from '../../shared/models/users' +import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) { const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { @@ -96,13 +96,18 @@ export { // --------------------------------------------------------------------------- function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Transaction | undefined) { - return UserNotificationSettingModel.create({ + const values: UserNotificationSetting & { userId: number } = { userId: user.id, newVideoFromSubscription: UserNotificationSettingValue.WEB_NOTIFICATION, newCommentOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION, myVideoImportFinished: UserNotificationSettingValue.WEB_NOTIFICATION, myVideoPublished: UserNotificationSettingValue.WEB_NOTIFICATION, videoAbuseAsModerator: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL, - blacklistOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL - }, { transaction: t }) + blacklistOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL, + newUserRegistration: UserNotificationSettingValue.WEB_NOTIFICATION, + commentMention: UserNotificationSettingValue.WEB_NOTIFICATION, + newFollow: UserNotificationSettingValue.WEB_NOTIFICATION + } + + return UserNotificationSettingModel.create(values, { transaction: t }) } -- cgit v1.2.3 From 2f1548fda32c3ba9e53913270394eedfacd55986 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 8 Jan 2019 11:26:41 +0100 Subject: Add notifications in the client --- server/lib/notifier.ts | 4 ++-- server/lib/user.ts | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'server/lib') diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts index 2c51d7101..d1b331346 100644 --- a/server/lib/notifier.ts +++ b/server/lib/notifier.ts @@ -436,11 +436,11 @@ class Notifier { private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) { if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified !== true) return false - return value === UserNotificationSettingValue.EMAIL || value === UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL + return value & UserNotificationSettingValue.EMAIL } private isWebNotificationEnabled (value: UserNotificationSettingValue) { - return value === UserNotificationSettingValue.WEB_NOTIFICATION || value === UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL + return value & UserNotificationSettingValue.WEB } static get Instance () { diff --git a/server/lib/user.ts b/server/lib/user.ts index 9e24e85a0..a39ef6c3d 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -98,15 +98,15 @@ export { function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Transaction | undefined) { const values: UserNotificationSetting & { userId: number } = { userId: user.id, - newVideoFromSubscription: UserNotificationSettingValue.WEB_NOTIFICATION, - newCommentOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION, - myVideoImportFinished: UserNotificationSettingValue.WEB_NOTIFICATION, - myVideoPublished: UserNotificationSettingValue.WEB_NOTIFICATION, - videoAbuseAsModerator: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL, - blacklistOnMyVideo: UserNotificationSettingValue.WEB_NOTIFICATION_AND_EMAIL, - newUserRegistration: UserNotificationSettingValue.WEB_NOTIFICATION, - commentMention: UserNotificationSettingValue.WEB_NOTIFICATION, - newFollow: UserNotificationSettingValue.WEB_NOTIFICATION + newVideoFromSubscription: UserNotificationSettingValue.WEB, + newCommentOnMyVideo: UserNotificationSettingValue.WEB, + myVideoImportFinished: UserNotificationSettingValue.WEB, + myVideoPublished: UserNotificationSettingValue.WEB, + videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newUserRegistration: UserNotificationSettingValue.WEB, + commentMention: UserNotificationSettingValue.WEB, + newFollow: UserNotificationSettingValue.WEB } return UserNotificationSettingModel.create(values, { transaction: t }) -- cgit v1.2.3 From a4101923e699e49ceb9ff36e971c75417fafc9f0 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 9 Jan 2019 15:14:29 +0100 Subject: Implement contact form on server side --- server/lib/emailer.ts | 23 +++++++++++++++++++++-- server/lib/job-queue/handlers/email.ts | 3 ++- server/lib/redis.ts | 24 ++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 5 deletions(-) (limited to 'server/lib') diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 3429498e7..9b1c5122f 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -354,13 +354,32 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - sendMail (to: string[], subject: string, text: string) { + addContactFormJob (fromEmail: string, fromName: string, body: string) { + const text = 'Hello dear admin,\n\n' + + fromName + ' sent you a message' + + '\n\n---------------------------------------\n\n' + + body + + '\n\n---------------------------------------\n\n' + + 'Cheers,\n' + + 'PeerTube.' + + const emailPayload: EmailPayload = { + from: fromEmail, + to: [ CONFIG.ADMIN.EMAIL ], + subject: '[PeerTube] Contact form submitted', + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + + sendMail (to: string[], subject: string, text: string, from?: string) { if (!this.enabled) { throw new Error('Cannot send mail because SMTP is not configured.') } return this.transporter.sendMail({ - from: CONFIG.SMTP.FROM_ADDRESS, + from: from || CONFIG.SMTP.FROM_ADDRESS, to: to.join(','), subject, text diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts index 73d98ae54..220d0af32 100644 --- a/server/lib/job-queue/handlers/email.ts +++ b/server/lib/job-queue/handlers/email.ts @@ -6,13 +6,14 @@ export type EmailPayload = { to: string[] subject: string text: string + from?: string } async function processEmail (job: Bull.Job) { const payload = job.data as EmailPayload logger.info('Processing email in job %d.', job.id) - return Emailer.Instance.sendMail(payload.to, payload.subject, payload.text) + return Emailer.Instance.sendMail(payload.to, payload.subject, payload.text, payload.from) } // --------------------------------------------------------------------------- diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 3e25e6a2c..3628c0583 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -2,7 +2,13 @@ import * as express from 'express' import { createClient, RedisClient } from 'redis' import { logger } from '../helpers/logger' import { generateRandomString } from '../helpers/utils' -import { CONFIG, USER_PASSWORD_RESET_LIFETIME, USER_EMAIL_VERIFY_LIFETIME, VIDEO_VIEW_LIFETIME } from '../initializers' +import { + CONFIG, + CONTACT_FORM_LIFETIME, + USER_EMAIL_VERIFY_LIFETIME, + USER_PASSWORD_RESET_LIFETIME, + VIDEO_VIEW_LIFETIME +} from '../initializers' type CachedRoute = { body: string, @@ -76,6 +82,16 @@ class Redis { return this.getValue(this.generateVerifyEmailKey(userId)) } + /************* Contact form per IP *************/ + + async setContactFormIp (ip: string) { + return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) + } + + async isContactFormIpExists (ip: string) { + return this.exists(this.generateContactFormKey(ip)) + } + /************* Views per IP *************/ setIPVideoView (ip: string, videoUUID: string) { @@ -175,7 +191,11 @@ class Redis { } private generateViewKey (ip: string, videoUUID: string) { - return videoUUID + '-' + ip + return `views-${videoUUID}-${ip}` + } + + private generateContactFormKey (ip: string) { + return 'contact-form-' + ip } /************* Redis helpers *************/ -- cgit v1.2.3 From d3e56c0c4b307c99e83fbafb7f2c5884cbc20055 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 10 Jan 2019 11:12:41 +0100 Subject: Implement contact form in the client --- server/lib/emailer.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'server/lib') diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 9b1c5122f..f384a254e 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -18,7 +18,6 @@ class Emailer { private static instance: Emailer private initialized = false private transporter: Transporter - private enabled = false private constructor () {} @@ -27,7 +26,7 @@ class Emailer { if (this.initialized === true) return this.initialized = true - if (CONFIG.SMTP.HOSTNAME && CONFIG.SMTP.PORT) { + if (Emailer.isEnabled()) { logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) let tls @@ -55,8 +54,6 @@ class Emailer { tls, auth }) - - this.enabled = true } else { if (!isTestInstance()) { logger.error('Cannot use SMTP server because of lack of configuration. PeerTube will not be able to send mails!') @@ -64,8 +61,8 @@ class Emailer { } } - isEnabled () { - return this.enabled + static isEnabled () { + return !!CONFIG.SMTP.HOSTNAME && !!CONFIG.SMTP.PORT } async checkConnectionOrDie () { @@ -374,7 +371,7 @@ class Emailer { } sendMail (to: string[], subject: string, text: string, from?: string) { - if (!this.enabled) { + if (!Emailer.isEnabled()) { throw new Error('Cannot send mail because SMTP is not configured.') } -- cgit v1.2.3 From 5abb9fbbd12e7097e348d6a38622d364b1fa47ed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 10 Jan 2019 15:39:51 +0100 Subject: Add ability to unfederate a local video (on blacklist) --- server/lib/activitypub/share.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 5dcba778c..170e49238 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -78,7 +78,7 @@ async function shareByServer (video: VideoModel, t: Transaction) { const serverActor = await getServerActor() const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video) - return VideoShareModel.findOrCreate({ + const [ serverShare ] = await VideoShareModel.findOrCreate({ defaults: { actorId: serverActor.id, videoId: video.id, @@ -88,16 +88,14 @@ async function shareByServer (video: VideoModel, t: Transaction) { url: serverShareUrl }, transaction: t - }).then(([ serverShare, created ]) => { - if (created) return sendVideoAnnounce(serverActor, serverShare, video, t) - - return undefined }) + + return sendVideoAnnounce(serverActor, serverShare, video, t) } async function shareByVideoChannel (video: VideoModel, t: Transaction) { const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) - return VideoShareModel.findOrCreate({ + const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ defaults: { actorId: video.VideoChannel.actorId, videoId: video.id, @@ -107,11 +105,9 @@ async function shareByVideoChannel (video: VideoModel, t: Transaction) { url: videoChannelShareUrl }, transaction: t - }).then(([ videoChannelShare, created ]) => { - if (created) return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) - - return undefined }) + + return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) } async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) { -- cgit v1.2.3 From c04eb647db4f543a31a8100c1ec9a86c700bca6a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 10 Jan 2019 16:00:23 +0100 Subject: Use origin video url in canonical tag --- server/lib/client-html.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'server/lib') diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 1875ec1fc..b2c376e20 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -1,7 +1,7 @@ import * as express from 'express' import * as Bluebird from 'bluebird' import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' -import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, STATIC_PATHS } from '../initializers' +import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE } from '../initializers' import { join } from 'path' import { escapeHTML } from '../helpers/core-utils' import { VideoModel } from '../models/video/video' @@ -187,8 +187,8 @@ export class ClientHtml { // Schema.org tagsString += `` - // SEO - tagsString += `` + // SEO, use origin video url so Google does not index remote videos + tagsString += `` return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) } -- cgit v1.2.3 From b4593cd7ff34b94b60f6bfa0b57e371d74d63aa2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 10:24:49 +0100 Subject: Warn user when they want to delete a channel Because they will not be able to create another channel with the same actor name --- server/lib/activitypub/actor.ts | 2 +- server/lib/activitypub/process/process.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f7bf7c65a..f80296725 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -296,7 +296,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe const actorJSON: ActivityPubActor = requestResult.body if (isActorObjectValid(actorJSON) === false) { - logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON }) + logger.debug('Remote actor JSON is not valid.', { actorJSON }) return { result: undefined, statusCode: requestResult.response.statusCode } } diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index bcc5cac7a..2479d5da2 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -35,7 +35,7 @@ async function processActivities ( const actorsCache: { [ url: string ]: ActorModel } = {} for (const activity of activities) { - if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { + if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) continue } -- cgit v1.2.3 From 744d0eca195bce7dafeb4a958d0eb3c0046be32d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 11:30:15 +0100 Subject: Refresh remote actors on GET enpoints --- server/lib/activitypub/actor.ts | 111 +++++++++++---------- server/lib/activitypub/videos.ts | 2 +- .../job-queue/handlers/activitypub-refresher.ts | 25 +++-- 3 files changed, 77 insertions(+), 61 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f80296725..d728c81d1 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -201,6 +201,62 @@ async function addFetchOutboxJob (actor: ActorModel) { return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) } +async function refreshActorIfNeeded ( + actorArg: ActorModel, + fetchedType: ActorFetchByUrlType +): Promise<{ actor: ActorModel, refreshed: boolean }> { + if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } + + // We need more attributes + const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) + + try { + const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + const { result, statusCode } = await fetchRemoteActor(actorUrl) + + if (statusCode === 404) { + logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) + actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() + return { actor: undefined, refreshed: false } + } + + if (result === undefined) { + logger.warn('Cannot fetch remote actor in refresh actor.') + return { actor, refreshed: false } + } + + return sequelizeTypescript.transaction(async t => { + updateInstanceWithAnother(actor, result.actor) + + if (result.avatarName !== undefined) { + await updateActorAvatarInstance(actor, result.avatarName, t) + } + + // Force update + actor.setDataValue('updatedAt', new Date()) + await actor.save({ transaction: t }) + + if (actor.Account) { + actor.Account.set('name', result.name) + actor.Account.set('description', result.summary) + + await actor.Account.save({ transaction: t }) + } else if (actor.VideoChannel) { + actor.VideoChannel.set('name', result.name) + actor.VideoChannel.set('description', result.summary) + actor.VideoChannel.set('support', result.support) + + await actor.VideoChannel.save({ transaction: t }) + } + + return { refreshed: true, actor } + }) + } catch (err) { + logger.warn('Cannot refresh actor.', { err }) + return { actor, refreshed: false } + } +} + export { getOrCreateActorAndServerAndModel, buildActorInstance, @@ -208,6 +264,7 @@ export { fetchActorTotalItems, fetchAvatarIfExists, updateActorInstance, + refreshActorIfNeeded, updateActorAvatarInstance, addFetchOutboxJob } @@ -373,58 +430,4 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannelCreated } -async function refreshActorIfNeeded ( - actorArg: ActorModel, - fetchedType: ActorFetchByUrlType -): Promise<{ actor: ActorModel, refreshed: boolean }> { - if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } - - // We need more attributes - const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) - - try { - const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) - const { result, statusCode } = await fetchRemoteActor(actorUrl) - - if (statusCode === 404) { - logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) - actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() - return { actor: undefined, refreshed: false } - } - - if (result === undefined) { - logger.warn('Cannot fetch remote actor in refresh actor.') - return { actor, refreshed: false } - } - return sequelizeTypescript.transaction(async t => { - updateInstanceWithAnother(actor, result.actor) - - if (result.avatarName !== undefined) { - await updateActorAvatarInstance(actor, result.avatarName, t) - } - - // Force update - actor.setDataValue('updatedAt', new Date()) - await actor.save({ transaction: t }) - - if (actor.Account) { - actor.Account.set('name', result.name) - actor.Account.set('description', result.summary) - - await actor.Account.save({ transaction: t }) - } else if (actor.VideoChannel) { - actor.VideoChannel.set('name', result.name) - actor.VideoChannel.set('description', result.summary) - actor.VideoChannel.set('support', result.support) - - await actor.VideoChannel.save({ transaction: t }) - } - - return { refreshed: true, actor } - }) - } catch (err) { - logger.warn('Cannot refresh actor.', { err }) - return { actor, refreshed: false } - } -} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 893768769..cbdd981c5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -179,7 +179,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { } if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) - else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) + else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } }) } return { video: videoFromDatabase, created: false } diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 671b0f487..454b975fe 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -1,30 +1,33 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { fetchVideoByUrl } from '../../../helpers/video' -import { refreshVideoIfNeeded } from '../../activitypub' +import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' +import { ActorModel } from '../../../models/activitypub/actor' export type RefreshPayload = { - videoUrl: string - type: 'video' + type: 'video' | 'actor' + url: string } async function refreshAPObject (job: Bull.Job) { const payload = job.data as RefreshPayload - logger.info('Processing AP refresher in job %d for video %s.', job.id, payload.videoUrl) + logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) - if (payload.type === 'video') return refreshAPVideo(payload.videoUrl) + if (payload.type === 'video') return refreshVideo(payload.url) + if (payload.type === 'actor') return refreshActor(payload.url) } // --------------------------------------------------------------------------- export { + refreshActor, refreshAPObject } // --------------------------------------------------------------------------- -async function refreshAPVideo (videoUrl: string) { +async function refreshVideo (videoUrl: string) { const fetchType = 'all' as 'all' const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } @@ -39,3 +42,13 @@ async function refreshAPVideo (videoUrl: string) { await refreshVideoIfNeeded(refreshOptions) } } + +async function refreshActor (actorUrl: string) { + const fetchType = 'all' as 'all' + const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) + + if (actor) { + await refreshActorIfNeeded(actor, fetchType) + } + +} -- cgit v1.2.3 From 699b059e2d6cdd09685a69261f2ca5cf63053a71 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 12:11:06 +0100 Subject: Fix deleting not found remote actors --- server/lib/activitypub/actor.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index d728c81d1..edf38bc0a 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -211,7 +211,14 @@ async function refreshActorIfNeeded ( const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) try { - const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + let actorUrl: string + try { + actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + } catch (err) { + logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) + actorUrl = actor.url + } + const { result, statusCode } = await fetchRemoteActor(actorUrl) if (statusCode === 404) { @@ -429,5 +436,3 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannelCreated } - - -- cgit v1.2.3 From 848f499def54db2dd36437ef0dfb74dd5041c23b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 15 Jan 2019 11:14:12 +0100 Subject: Prepare Dislike/Flag/View fixes For now we Create these activities, but we should just send them directly. This fix handles correctly direct Dislikes/Flags/Views, we'll implement the sending correctly these activities in the next peertube version --- server/lib/activitypub/actor.ts | 4 +- server/lib/activitypub/process/process-accept.ts | 1 - server/lib/activitypub/process/process-create.ts | 118 +++++----------------- server/lib/activitypub/process/process-dislike.ts | 52 ++++++++++ server/lib/activitypub/process/process-flag.ts | 49 +++++++++ server/lib/activitypub/process/process-follow.ts | 3 +- server/lib/activitypub/process/process-like.ts | 3 +- server/lib/activitypub/process/process-undo.ts | 8 +- server/lib/activitypub/process/process-view.ts | 35 +++++++ server/lib/activitypub/process/process.ts | 12 ++- server/lib/activitypub/share.ts | 4 +- server/lib/activitypub/video-rates.ts | 4 +- server/lib/activitypub/videos.ts | 6 +- 13 files changed, 191 insertions(+), 108 deletions(-) create mode 100644 server/lib/activitypub/process/process-dislike.ts create mode 100644 server/lib/activitypub/process/process-flag.ts create mode 100644 server/lib/activitypub/process/process-view.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index edf38bc0a..8215840da 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -4,7 +4,7 @@ import * as url from 'url' import * as uuidv4 from 'uuid/v4' import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' @@ -42,7 +42,7 @@ async function getOrCreateActorAndServerAndModel ( recurseIfNeeded = true, updateCollections = false ) { - const actorUrl = getAPUrl(activityActor) + const actorUrl = getAPId(activityActor) let created = false let actor = await fetchActorByUrl(actorUrl, fetchType) diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 605705ad3..ebb275e34 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts @@ -2,7 +2,6 @@ import { ActivityAccept } from '../../../../shared/models/activitypub' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { addFetchOutboxJob } from '../actor' -import { Notifier } from '../../notifier' async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) { if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 2e04ee843..5f4d793a5 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -1,36 +1,44 @@ -import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared' -import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' +import { ActivityCreate, CacheFileObject, VideoTorrentObject } from '../../../../shared' import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' -import { VideoAbuseModel } from '../../../models/video/video-abuse' import { addVideoComment, resolveThread } from '../video-comments' import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardVideoRelatedActivity } from '../send/utils' -import { Redis } from '../../redis' import { createOrUpdateCacheFile } from '../cache-file' -import { getVideoDislikeActivityPubUrl } from '../url' import { Notifier } from '../../notifier' +import { processViewActivity } from './process-view' +import { processDislikeActivity } from './process-dislike' +import { processFlagActivity } from './process-flag' async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { const activityObject = activity.object const activityType = activityObject.type if (activityType === 'View') { - return processCreateView(byActor, activity) - } else if (activityType === 'Dislike') { - return retryTransactionWrapper(processCreateDislike, byActor, activity) - } else if (activityType === 'Video') { + return processViewActivity(activity, byActor) + } + + if (activityType === 'Dislike') { + return retryTransactionWrapper(processDislikeActivity, activity, byActor) + } + + if (activityType === 'Flag') { + return retryTransactionWrapper(processFlagActivity, activity, byActor) + } + + if (activityType === 'Video') { return processCreateVideo(activity) - } else if (activityType === 'Flag') { - return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject) - } else if (activityType === 'Note') { - return retryTransactionWrapper(processCreateVideoComment, byActor, activity) - } else if (activityType === 'CacheFile') { - return retryTransactionWrapper(processCacheFile, byActor, activity) + } + + if (activityType === 'Note') { + return retryTransactionWrapper(processCreateVideoComment, activity, byActor) + } + + if (activityType === 'CacheFile') { + return retryTransactionWrapper(processCacheFile, activity, byActor) } logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) @@ -55,56 +63,7 @@ async function processCreateVideo (activity: ActivityCreate) { return video } -async function processCreateDislike (byActor: ActorModel, activity: ActivityCreate) { - const dislike = activity.object as DislikeObject - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) - - return sequelizeTypescript.transaction(async t => { - const rate = { - type: 'dislike' as 'dislike', - videoId: video.id, - accountId: byAccount.id - } - - const [ , created ] = await AccountVideoRateModel.findOrCreate({ - where: rate, - defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), - transaction: t - }) - if (created === true) await video.increment('dislikes', { transaction: t }) - - if (video.isOwned() && created === true) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, t, exceptions, video) - } - }) -} - -async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { - const view = activity.object as ViewObject - - const options = { - videoObject: view.object, - fetchType: 'only-video' as 'only-video' - } - const { video } = await getOrCreateVideoAndAccountAndChannel(options) - - await Redis.Instance.addVideoView(video.id) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} - -async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { +async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) { const cacheFile = activity.object as CacheFileObject const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) @@ -120,32 +79,7 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) } } -async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { - logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) - - const account = byActor.Account - if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) - - return sequelizeTypescript.transaction(async t => { - const videoAbuseData = { - reporterAccountId: account.id, - reason: videoAbuseToCreateData.content, - videoId: video.id, - state: VideoAbuseState.PENDING - } - - const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) - videoAbuseInstance.Video = video - - Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) - - logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) - }) -} - -async function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreate) { +async function processCreateVideoComment (activity: ActivityCreate, byActor: ActorModel) { const commentObject = activity.object as VideoCommentObject const byAccount = byActor.Account diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts new file mode 100644 index 000000000..bfd69e07a --- /dev/null +++ b/server/lib/activitypub/process/process-dislike.ts @@ -0,0 +1,52 @@ +import { ActivityCreate, ActivityDislike } from '../../../../shared' +import { DislikeObject } from '../../../../shared/models/activitypub/objects' +import { retryTransactionWrapper } from '../../../helpers/database-utils' +import { sequelizeTypescript } from '../../../initializers' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate' +import { ActorModel } from '../../../models/activitypub/actor' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { forwardVideoRelatedActivity } from '../send/utils' +import { getVideoDislikeActivityPubUrl } from '../url' + +async function processDislikeActivity (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { + return retryTransactionWrapper(processDislike, activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processDislikeActivity +} + +// --------------------------------------------------------------------------- + +async function processDislike (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { + const dislikeObject = activity.type === 'Dislike' ? activity.object : (activity.object as DislikeObject).object + const byAccount = byActor.Account + + if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) + + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) + + return sequelizeTypescript.transaction(async t => { + const rate = { + type: 'dislike' as 'dislike', + videoId: video.id, + accountId: byAccount.id + } + + const [ , created ] = await AccountVideoRateModel.findOrCreate({ + where: rate, + defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), + transaction: t + }) + if (created === true) await video.increment('dislikes', { transaction: t }) + + if (video.isOwned() && created === true) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + + await forwardVideoRelatedActivity(activity, t, exceptions, video) + } + }) +} diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts new file mode 100644 index 000000000..79ce6fb41 --- /dev/null +++ b/server/lib/activitypub/process/process-flag.ts @@ -0,0 +1,49 @@ +import { ActivityCreate, ActivityFlag, VideoAbuseState } from '../../../../shared' +import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects' +import { retryTransactionWrapper } from '../../../helpers/database-utils' +import { logger } from '../../../helpers/logger' +import { sequelizeTypescript } from '../../../initializers' +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoAbuseModel } from '../../../models/video/video-abuse' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { Notifier } from '../../notifier' +import { getAPId } from '../../../helpers/activitypub' + +async function processFlagActivity (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { + return retryTransactionWrapper(processCreateVideoAbuse, activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processFlagActivity +} + +// --------------------------------------------------------------------------- + +async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { + const flag = activity.type === 'Flag' ? activity : (activity.object as VideoAbuseObject) + + logger.debug('Reporting remote abuse for video %s.', getAPId(flag.object)) + + const account = byActor.Account + if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) + + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object }) + + return sequelizeTypescript.transaction(async t => { + const videoAbuseData = { + reporterAccountId: account.id, + reason: flag.content, + videoId: video.id, + state: VideoAbuseState.PENDING + } + + const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) + videoAbuseInstance.Video = video + + Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) + + logger.info('Remote abuse for video uuid %s created', flag.object) + }) +} diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index a67892440..0cd537187 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts @@ -6,9 +6,10 @@ import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { sendAccept } from '../send' import { Notifier } from '../../notifier' +import { getAPId } from '../../../helpers/activitypub' async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { - const activityObject = activity.object + const activityObject = getAPId(activity.object) return retryTransactionWrapper(processFollow, byActor, activityObject) } diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index e8e97eece..2a04167d7 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -6,6 +6,7 @@ import { ActorModel } from '../../../models/activitypub/actor' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { getVideoLikeActivityPubUrl } from '../url' +import { getAPId } from '../../../helpers/activitypub' async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { return retryTransactionWrapper(processLikeVideo, byActor, activity) @@ -20,7 +21,7 @@ export { // --------------------------------------------------------------------------- async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { - const videoUrl = activity.object + const videoUrl = getAPId(activity.object) const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 438a013b6..ed0177a67 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -26,6 +26,10 @@ async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) } } + if (activityToUndo.type === 'Dislike') { + return retryTransactionWrapper(processUndoDislike, byActor, activity) + } + if (activityToUndo.type === 'Follow') { return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) } @@ -72,7 +76,9 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { } async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { - const dislike = activity.object.object as DislikeObject + const dislike = activity.object.type === 'Dislike' + ? activity.object + : activity.object.object as DislikeObject const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts new file mode 100644 index 000000000..8f66d3630 --- /dev/null +++ b/server/lib/activitypub/process/process-view.ts @@ -0,0 +1,35 @@ +import { ActorModel } from '../../../models/activitypub/actor' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { forwardVideoRelatedActivity } from '../send/utils' +import { Redis } from '../../redis' +import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' + +async function processViewActivity (activity: ActivityView | ActivityCreate, byActor: ActorModel) { + return processCreateView(activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processViewActivity +} + +// --------------------------------------------------------------------------- + +async function processCreateView (activity: ActivityView | ActivityCreate, byActor: ActorModel) { + const videoObject = activity.type === 'View' ? activity.object : (activity.object as ViewObject).object + + const options = { + videoObject: videoObject, + fetchType: 'only-video' as 'only-video' + } + const { video } = await getOrCreateVideoAndAccountAndChannel(options) + + await Redis.Instance.addVideoView(video.id) + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + await forwardVideoRelatedActivity(activity, undefined, exceptions, video) + } +} diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index 2479d5da2..9dd241402 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -1,5 +1,5 @@ import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { checkUrlsSameHost, getAPUrl } from '../../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/activitypub/actor' import { processAcceptActivity } from './process-accept' @@ -12,6 +12,9 @@ import { processRejectActivity } from './process-reject' import { processUndoActivity } from './process-undo' import { processUpdateActivity } from './process-update' import { getOrCreateActorAndServerAndModel } from '../actor' +import { processDislikeActivity } from './process-dislike' +import { processFlagActivity } from './process-flag' +import { processViewActivity } from './process-view' const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise } = { Create: processCreateActivity, @@ -22,7 +25,10 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: Ac Reject: processRejectActivity, Announce: processAnnounceActivity, Undo: processUndoActivity, - Like: processLikeActivity + Like: processLikeActivity, + Dislike: processDislikeActivity, + Flag: processFlagActivity, + View: processViewActivity } async function processActivities ( @@ -40,7 +46,7 @@ async function processActivities ( continue } - const actorUrl = getAPUrl(activity.actor) + const actorUrl = getAPId(activity.actor) // When we fetch remote data, we don't have signature if (options.signatureActor && actorUrl !== options.signatureActor.url) { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 170e49238..1767df0ae 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -11,7 +11,7 @@ import { doRequest } from '../../helpers/requests' import { getOrCreateActorAndServerAndModel } from './actor' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -41,7 +41,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') - const actorUrl = getAPUrl(body.actor) + const actorUrl = getAPId(body.actor) if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) } diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 2cce67f0c..45a2b22ea 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' import { doRequest } from '../../helpers/requests' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { ActorModel } from '../../models/activitypub/actor' import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' @@ -26,7 +26,7 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') - const actorUrl = getAPUrl(body.actor) + const actorUrl = getAPId(body.actor) if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index cbdd981c5..e1e523499 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -28,7 +28,7 @@ import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { Notifier } from '../notifier' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { @@ -155,7 +155,7 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid } async function getOrCreateVideoAndAccountAndChannel (options: { - videoObject: VideoTorrentObject | string, + videoObject: { id: string } | string, syncParam?: SyncParam, fetchType?: VideoFetchByUrlType, allowRefresh?: boolean // true by default @@ -166,7 +166,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { const allowRefresh = options.allowRefresh !== false // Get video url - const videoUrl = getAPUrl(options.videoObject) + const videoUrl = getAPId(options.videoObject) let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) if (videoFromDatabase) { -- cgit v1.2.3 From 1e7eb25f6cb6893db8f99ff40ef0509aa2a16614 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 15 Jan 2019 14:52:33 +0100 Subject: Correctly send Flag/Dislike/View activities --- server/lib/activitypub/send/send-create.ts | 69 ----------------------------- server/lib/activitypub/send/send-dislike.ts | 41 +++++++++++++++++ server/lib/activitypub/send/send-flag.ts | 39 ++++++++++++++++ server/lib/activitypub/send/send-undo.ts | 12 ++--- server/lib/activitypub/send/send-view.ts | 40 +++++++++++++++++ server/lib/activitypub/video-rates.ts | 5 ++- 6 files changed, 129 insertions(+), 77 deletions(-) create mode 100644 server/lib/activitypub/send/send-dislike.ts create mode 100644 server/lib/activitypub/send/send-flag.ts create mode 100644 server/lib/activitypub/send/send-view.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index e3fca0a17..73e667ad4 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -3,9 +3,7 @@ import { ActivityAudience, ActivityCreate } from '../../../../shared/models/acti import { VideoPrivacy } from '../../../../shared/models/videos' import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' -import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' -import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' import { logger } from '../../../helpers/logger' @@ -25,20 +23,6 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { return broadcastToFollowers(createActivity, byActor, [ byActor ], t) } -async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { - if (!video.VideoChannel.Account.Actor.serverId) return // Local - - const url = getVideoAbuseActivityPubUrl(videoAbuse) - - logger.info('Creating job to send video abuse %s.', url) - - // Custom audience, we only send the abuse to the origin instance - const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } - const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) - - return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) -} - async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { logger.info('Creating job to send file cache of %s.', fileRedundancy.url) @@ -91,37 +75,6 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) } -async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { - logger.info('Creating job to send view of %s.', video.url) - - const url = getVideoViewActivityPubUrl(byActor, video) - const viewActivity = buildViewActivity(url, byActor, video) - - return sendVideoRelatedCreateActivity({ - // Use the server actor to send the view - byActor, - video, - url, - object: viewActivity, - transaction: t - }) -} - -async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { - logger.info('Creating job to dislike %s.', video.url) - - const url = getVideoDislikeActivityPubUrl(byActor, video) - const dislikeActivity = buildDislikeActivity(url, byActor, video) - - return sendVideoRelatedCreateActivity({ - byActor, - video, - url, - object: dislikeActivity, - transaction: t - }) -} - function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { if (!audience) audience = getAudience(byActor) @@ -136,33 +89,11 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud ) } -function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel) { - return { - id: url, - type: 'Dislike', - actor: byActor.url, - object: video.url - } -} - -function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel) { - return { - id: url, - type: 'View', - actor: byActor.url, - object: video.url - } -} - // --------------------------------------------------------------------------- export { sendCreateVideo, - sendVideoAbuse, buildCreateActivity, - sendCreateView, - sendCreateDislike, - buildDislikeActivity, sendCreateVideoComment, sendCreateCacheFile } diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts new file mode 100644 index 000000000..a88436f2c --- /dev/null +++ b/server/lib/activitypub/send/send-dislike.ts @@ -0,0 +1,41 @@ +import { Transaction } from 'sequelize' +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoModel } from '../../../models/video/video' +import { getVideoDislikeActivityPubUrl } from '../url' +import { logger } from '../../../helpers/logger' +import { ActivityAudience, ActivityDislike } from '../../../../shared/models/activitypub' +import { sendVideoRelatedActivity } from './utils' +import { audiencify, getAudience } from '../audience' + +async function sendDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { + logger.info('Creating job to dislike %s.', video.url) + + const activityBuilder = (audience: ActivityAudience) => { + const url = getVideoDislikeActivityPubUrl(byActor, video) + + return buildDislikeActivity(url, byActor, video, audience) + } + + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) +} + +function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityDislike { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + id: url, + type: 'Dislike' as 'Dislike', + actor: byActor.url, + object: video.url + }, + audience + ) +} + +// --------------------------------------------------------------------------- + +export { + sendDislike, + buildDislikeActivity +} diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts new file mode 100644 index 000000000..96a7311b9 --- /dev/null +++ b/server/lib/activitypub/send/send-flag.ts @@ -0,0 +1,39 @@ +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoModel } from '../../../models/video/video' +import { VideoAbuseModel } from '../../../models/video/video-abuse' +import { getVideoAbuseActivityPubUrl } from '../url' +import { unicastTo } from './utils' +import { logger } from '../../../helpers/logger' +import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub' +import { audiencify, getAudience } from '../audience' + +async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { + if (!video.VideoChannel.Account.Actor.serverId) return // Local user + + const url = getVideoAbuseActivityPubUrl(videoAbuse) + + logger.info('Creating job to send video abuse %s.', url) + + // Custom audience, we only send the abuse to the origin instance + const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } + const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience) + + return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) +} + +function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag { + if (!audience) audience = getAudience(byActor) + + const activity = Object.assign( + { id: url, actor: byActor.url }, + videoAbuse.toActivityPubObject() + ) + + return audiencify(activity, audience) +} + +// --------------------------------------------------------------------------- + +export { + sendVideoAbuse +} diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index bf1b6e117..eb18a6cb6 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -2,7 +2,7 @@ import { Transaction } from 'sequelize' import { ActivityAnnounce, ActivityAudience, - ActivityCreate, + ActivityCreate, ActivityDislike, ActivityFollow, ActivityLike, ActivityUndo @@ -13,13 +13,14 @@ import { VideoModel } from '../../../models/video/video' import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' import { audiencify, getAudience } from '../audience' -import { buildCreateActivity, buildDislikeActivity } from './send-create' +import { buildCreateActivity } from './send-create' import { buildFollowActivity } from './send-follow' import { buildLikeActivity } from './send-like' import { VideoShareModel } from '../../../models/video/video-share' import { buildAnnounceWithVideoAudience } from './send-announce' import { logger } from '../../../helpers/logger' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' +import { buildDislikeActivity } from './send-dislike' async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { const me = actorFollow.ActorFollower @@ -65,9 +66,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) - const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) - return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) + return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) } async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { @@ -94,7 +94,7 @@ export { function undoActivityData ( url: string, byActor: ActorModel, - object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, + object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, audience?: ActivityAudience ): ActivityUndo { if (!audience) audience = getAudience(byActor) @@ -114,7 +114,7 @@ async function sendUndoVideoRelatedActivity (options: { byActor: ActorModel, video: VideoModel, url: string, - activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, + activity: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, transaction: Transaction }) { const activityBuilder = (audience: ActivityAudience) => { diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts new file mode 100644 index 000000000..8ad126be0 --- /dev/null +++ b/server/lib/activitypub/send/send-view.ts @@ -0,0 +1,40 @@ +import { Transaction } from 'sequelize' +import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoModel } from '../../../models/video/video' +import { getVideoLikeActivityPubUrl } from '../url' +import { sendVideoRelatedActivity } from './utils' +import { audiencify, getAudience } from '../audience' +import { logger } from '../../../helpers/logger' + +async function sendView (byActor: ActorModel, video: VideoModel, t: Transaction) { + logger.info('Creating job to send view of %s.', video.url) + + const activityBuilder = (audience: ActivityAudience) => { + const url = getVideoLikeActivityPubUrl(byActor, video) + + return buildViewActivity(url, byActor, video, audience) + } + + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) +} + +function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityView { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + id: url, + type: 'View' as 'View', + actor: byActor.url, + object: video.url + }, + audience + ) +} + +// --------------------------------------------------------------------------- + +export { + sendView +} diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 45a2b22ea..7aac79118 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -1,7 +1,7 @@ import { Transaction } from 'sequelize' import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' -import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' +import { sendLike, sendUndoDislike, sendUndoLike } from './send' import { VideoRateType } from '../../../shared/models/videos' import * as Bluebird from 'bluebird' import { getOrCreateActorAndServerAndModel } from './actor' @@ -12,6 +12,7 @@ import { doRequest } from '../../helpers/requests' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { ActorModel } from '../../models/activitypub/actor' import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' +import { sendDislike } from './send/send-dislike' async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { let rateCounts = 0 @@ -82,7 +83,7 @@ async function sendVideoRateChange (account: AccountModel, // Like if (likes > 0) await sendLike(actor, video, t) // Dislike - if (dislikes > 0) await sendCreateDislike(actor, video, t) + if (dislikes > 0) await sendDislike(actor, video, t) } function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { -- cgit v1.2.3 From f7effe8dc7c641388f7edbcaad716fc16321d794 Mon Sep 17 00:00:00 2001 From: Josh Morel Date: Wed, 6 Feb 2019 06:14:45 -0500 Subject: don't notify prior to scheduled update also increase timeouts on user-notification test --- server/lib/job-queue/handlers/video-file.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 593e43cc5..217d666b6 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -91,7 +91,8 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { return { videoDatabase, videoPublished } }) - if (videoPublished) { + // don't notify prior to scheduled video update + if (videoPublished && !videoDatabase.ScheduleVideoUpdate) { Notifier.Instance.notifyOnNewVideo(videoDatabase) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } @@ -149,8 +150,11 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo return { videoDatabase, videoPublished } }) - if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) - if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) + // don't notify prior to scheduled video update + if (!videoDatabase.ScheduleVideoUpdate) { + if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) + } } // --------------------------------------------------------------------------- -- cgit v1.2.3 From 092092969633bbcf6d4891a083ea497a7d5c3154 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 29 Jan 2019 08:37:25 +0100 Subject: Add hls support on server --- server/lib/activitypub/cache-file.ts | 23 ++- server/lib/activitypub/send/send-create.ts | 9 +- server/lib/activitypub/send/send-undo.ts | 3 +- server/lib/activitypub/send/send-update.ts | 2 +- server/lib/activitypub/url.ts | 7 + server/lib/activitypub/videos.ts | 97 ++++++++++- server/lib/hls.ts | 110 ++++++++++++ server/lib/job-queue/handlers/video-file.ts | 59 +++++-- .../lib/schedulers/videos-redundancy-scheduler.ts | 189 ++++++++++++++------- server/lib/video-transcoding.ts | 49 +++++- 10 files changed, 456 insertions(+), 92 deletions(-) create mode 100644 server/lib/hls.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index f6f068b45..9a40414bb 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -1,11 +1,28 @@ -import { CacheFileObject } from '../../../shared/index' +import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index' import { VideoModel } from '../../models/video/video' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { Transaction } from 'sequelize' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { - const url = cacheFileObject.url + if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { + const url = cacheFileObject.url + + const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) + if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) + + return { + expiresOn: new Date(cacheFileObject.expires), + url: cacheFileObject.id, + fileUrl: url.href, + strategy: null, + videoStreamingPlaylistId: playlist.id, + actorId: byActor.id + } + } + + const url = cacheFileObject.url const videoFile = video.VideoFiles.find(f => { return f.resolution === url.height && f.fps === url.fps }) @@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject return { expiresOn: new Date(cacheFileObject.expires), url: cacheFileObject.id, - fileUrl: cacheFileObject.url.href, + fileUrl: url.href, strategy: null, videoFileId: videoFile.id, actorId: byActor.id diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index e3fca0a17..605aaba06 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -1,6 +1,6 @@ import { Transaction } from 'sequelize' import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' -import { VideoPrivacy } from '../../../../shared/models/videos' +import { Video, VideoPrivacy } from '../../../../shared/models/videos' import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' import { VideoAbuseModel } from '../../../models/video/video-abuse' @@ -39,17 +39,14 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } -async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { +async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) { logger.info('Creating job to send file cache of %s.', fileRedundancy.url) - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) - const redundancyObject = fileRedundancy.toActivityPubObject() - return sendVideoRelatedCreateActivity({ byActor, video, url: fileRedundancy.url, - object: redundancyObject + object: fileRedundancy.toActivityPubObject() }) } diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index bf1b6e117..8976fcbc8 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { logger.info('Creating job to undo cache file %s.', redundancyModel.url) - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) + const videoId = redundancyModel.getVideo().id + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index a68f03edf..839f66470 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts @@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { logger.info('Creating job to update cache file %s.', redundancyModel.url) - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id) const activityBuilder = (audience: ActivityAudience) => { const redundancyObject = redundancyModel.toActivityPubObject() diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 38f15448c..4229fe094 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts @@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video' import { VideoAbuseModel } from '../../models/video/video-abuse' import { VideoCommentModel } from '../../models/video/video-comment' import { VideoFileModel } from '../../models/video/video-file' +import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' function getVideoActivityPubUrl (video: VideoModel) { return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid @@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` } +function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) { + return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}` +} + function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id } @@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) { export { getVideoActivityPubUrl, + getVideoCacheStreamingPlaylistActivityPubUrl, getVideoChannelActivityPubUrl, getAccountActivityPubUrl, getVideoAbuseActivityPubUrl, diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index e1e523499..edd01234f 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird' import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' import * as request from 'request' -import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' +import { + ActivityIconObject, + ActivityPlaylistSegmentHashesObject, + ActivityPlaylistUrlObject, + ActivityUrlObject, + ActivityVideoUrlObject, + VideoState +} from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' @@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { Notifier } from '../notifier' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' +import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -263,6 +273,25 @@ async function updateVideoFromAP (options: { options.video.VideoFiles = await Promise.all(upsertTasks) } + { + const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) + const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) + + // Remove video files that do not exist anymore + const destroyTasks = options.video.VideoStreamingPlaylists + .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) + .map(f => f.destroy(sequelizeOptions)) + await Promise.all(destroyTasks) + + // Update or add other one + const upsertTasks = streamingPlaylistAttributes.map(a => { + return VideoStreamingPlaylistModel.upsert(a, { returning: true, transaction: t }) + .then(([ streamingPlaylist ]) => streamingPlaylist) + }) + + options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks) + } + { // Update Tags const tags = options.videoObject.tag.map(tag => tag.name) @@ -367,13 +396,25 @@ export { // --------------------------------------------------------------------------- -function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { +function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) const urlMediaType = url.mediaType || url.mimeType return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') } +function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { + const urlMediaType = url.mediaType || url.mimeType + + return urlMediaType === 'application/x-mpegURL' +} + +function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { + const urlMediaType = tag.mediaType || tag.mimeType + + return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' +} + async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { logger.debug('Adding remote video %s.', videoObject.id) @@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) await Promise.all(videoFilePromises) + const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) + const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) + await Promise.all(playlistPromises) + // Process tags - const tags = videoObject.tag.map(t => t.name) + const tags = videoObject.tag + .filter(t => t.type === 'Hashtag') + .map(t => t.name) const tagInstances = await TagModel.findOrCreateTags(tags, t) await videoCreated.$set('Tags', tagInstances, sequelizeOptions) @@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes ( } function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { - const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] + const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] if (fileUrls.length === 0) { throw new Error('Cannot find video files for ' + video.url) } - const attributes: VideoFileModel[] = [] + const attributes: FilteredModelAttributes[] = [] for (const fileUrl of fileUrls) { // Fetch associated magnet uri const magnet = videoObject.url.find(u => { @@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid size: fileUrl.size, videoId: video.id, fps: fileUrl.fps || -1 - } as VideoFileModel + } + + attributes.push(attribute) + } + + return attributes +} + +function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { + const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] + if (playlistUrls.length === 0) return [] + + const attributes: FilteredModelAttributes[] = [] + for (const playlistUrlObject of playlistUrls) { + const p2pMediaLoaderInfohashes = playlistUrlObject.tag + .filter(t => t.type === 'Infohash') + .map(t => t.name) + if (p2pMediaLoaderInfohashes.length === 0) { + logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject }) + continue + } + + const segmentsSha256UrlObject = playlistUrlObject.tag + .find(t => { + return isAPPlaylistSegmentHashesUrlObject(t) + }) as ActivityPlaylistSegmentHashesObject + if (!segmentsSha256UrlObject) { + logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) + continue + } + + const attribute = { + type: VideoStreamingPlaylistType.HLS, + playlistUrl: playlistUrlObject.href, + segmentsSha256Url: segmentsSha256UrlObject.href, + p2pMediaLoaderInfohashes, + videoId: video.id + } + attributes.push(attribute) } diff --git a/server/lib/hls.ts b/server/lib/hls.ts new file mode 100644 index 000000000..10db6c3c3 --- /dev/null +++ b/server/lib/hls.ts @@ -0,0 +1,110 @@ +import { VideoModel } from '../models/video/video' +import { basename, dirname, join } from 'path' +import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' +import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' +import { getVideoFileSize } from '../helpers/ffmpeg-utils' +import { sha256 } from '../helpers/core-utils' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' +import HLSDownloader from 'hlsdownloader' +import { logger } from '../helpers/logger' +import { parse } from 'url' + +async function updateMasterHLSPlaylist (video: VideoModel) { + const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] + const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) + + for (const file of video.VideoFiles) { + // If we did not generated a playlist for this resolution, skip + const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + if (await pathExists(filePlaylistPath) === false) continue + + const videoFilePath = video.getVideoFilePath(file) + + const size = await getVideoFileSize(videoFilePath) + + const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) + const resolution = `RESOLUTION=${size.width}x${size.height}` + + let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` + if (file.fps) line += ',FRAME-RATE=' + file.fps + + masterPlaylists.push(line) + masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + } + + await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') +} + +async function updateSha256Segments (video: VideoModel) { + const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + const files = await readdir(directory) + const json: { [filename: string]: string} = {} + + for (const file of files) { + if (file.endsWith('.ts') === false) continue + + const buffer = await readFile(join(directory, file)) + const filename = basename(file) + + json[filename] = sha256(buffer) + } + + const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + await outputJSON(outputPath, json) +} + +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { + let timer + + logger.info('Importing HLS playlist %s', playlistUrl) + + const params = { + playlistURL: playlistUrl, + destination: CONFIG.STORAGE.TMP_DIR + } + const downloader = new HLSDownloader(params) + + const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname)) + + return new Promise(async (res, rej) => { + downloader.startDownload(err => { + clearTimeout(timer) + + if (err) { + deleteTmpDirectory(hlsDestinationDir) + + return rej(err) + } + + move(hlsDestinationDir, destinationDir, { overwrite: true }) + .then(() => res()) + .catch(err => { + deleteTmpDirectory(hlsDestinationDir) + + return rej(err) + }) + }) + + timer = setTimeout(() => { + deleteTmpDirectory(hlsDestinationDir) + + return rej(new Error('HLS download timeout.')) + }, timeout) + + function deleteTmpDirectory (directory: string) { + remove(directory) + .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + } + }) +} + +// --------------------------------------------------------------------------- + +export { + updateMasterHLSPlaylist, + updateSha256Segments, + downloadPlaylistSegments +} + +// --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 217d666b6..7119ce0ca 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video' import { JobQueue } from '../job-queue' import { federateVideoIfNeeded } from '../../activitypub' import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { sequelizeTypescript } from '../../../initializers' +import { sequelizeTypescript, CONFIG } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' -import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' +import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' import { Notifier } from '../../notifier' export type VideoFilePayload = { videoUUID: string - isNewVideo?: boolean resolution?: VideoResolution + isNewVideo?: boolean isPortraitMode?: boolean + generateHlsPlaylist?: boolean } export type VideoFileImportPayload = { @@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) { return undefined } - // Transcoding in other resolution - if (payload.resolution) { + if (payload.generateHlsPlaylist) { + await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) + + await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) + } else if (payload.resolution) { // Transcoding in other resolution await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) - await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) + await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload) } else { await optimizeVideofile(video) - await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) + await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) } return video } -async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { +async function onHlsPlaylistGenerationSuccess (video: VideoModel) { + if (video === undefined) return undefined + + await sequelizeTypescript.transaction(async t => { + // Maybe the video changed in database, refresh it + let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) + // Video does not exist anymore + if (!videoDatabase) return undefined + + // If the video was not published, we consider it is a new one for other instances + await federateVideoIfNeeded(videoDatabase, false, t) + }) +} + +async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) { if (video === undefined) return undefined const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { @@ -96,9 +114,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { Notifier.Instance.notifyOnNewVideo(videoDatabase) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } + + await createHlsJobIfEnabled(payload) } -async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { +async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) { if (videoArg === undefined) return undefined // Outside the transaction (IO on disk) @@ -145,7 +165,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) } - await federateVideoIfNeeded(videoDatabase, isNewVideo, t) + await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) return { videoDatabase, videoPublished } }) @@ -155,6 +175,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } + + await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) } // --------------------------------------------------------------------------- @@ -163,3 +185,20 @@ export { processVideoFile, processVideoFileImport } + +// --------------------------------------------------------------------------- + +function createHlsJobIfEnabled (payload?: VideoFilePayload) { + // Generate HLS playlist? + if (payload && CONFIG.TRANSCODING.HLS.ENABLED) { + const hlsTranscodingPayload = { + videoUUID: payload.videoUUID, + resolution: payload.resolution, + isPortraitMode: payload.isPortraitMode, + + generateHlsPlaylist: true + } + + return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload }) + } +} diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f643ee226..1a48f2bd0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,5 +1,5 @@ import { AbstractScheduler } from './abstract-scheduler' -import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' +import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' import { logger } from '../../helpers/logger' import { VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' @@ -9,9 +9,19 @@ import { join } from 'path' import { move } from 'fs-extra' import { getServerActor } from '../../helpers/utils' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' -import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' +import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' import { removeVideoRedundancy } from '../redundancy' import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' +import { VideoModel } from '../../models/video/video' +import { downloadPlaylistSegments } from '../hls' + +type CandidateToDuplicate = { + redundancy: VideosRedundancy, + video: VideoModel, + files: VideoFileModel[], + streamingPlaylists: VideoStreamingPlaylistModel[] +} export class VideosRedundancyScheduler extends AbstractScheduler { @@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } protected async internalExecute () { - for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { - logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) + for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { + logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) try { - const videoToDuplicate = await this.findVideoToDuplicate(obj) + const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) if (!videoToDuplicate) continue - const videoFiles = videoToDuplicate.VideoFiles - videoFiles.forEach(f => f.Video = videoToDuplicate) + const candidateToDuplicate = { + video: videoToDuplicate, + redundancy: redundancyConfig, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + } - await this.purgeCacheIfNeeded(obj, videoFiles) + await this.purgeCacheIfNeeded(candidateToDuplicate) - if (await this.isTooHeavy(obj, videoFiles)) { + if (await this.isTooHeavy(candidateToDuplicate)) { logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) continue } - logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) + logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) - await this.createVideoRedundancy(obj, videoFiles) + await this.createVideoRedundancies(candidateToDuplicate) } catch (err) { - logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) + logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) } } @@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { for (const redundancyModel of expired) { try { - await this.extendsOrDeleteRedundancy(redundancyModel) + const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) + const candidate = { + redundancy: redundancyConfig, + video: null, + files: [], + streamingPlaylists: [] + } + + // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it + if (!redundancyConfig || await this.isTooHeavy(candidate)) { + logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) + await removeVideoRedundancy(redundancyModel) + } else { + await this.extendsRedundancy(redundancyModel) + } } catch (err) { - logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) + logger.error( + 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), + { err } + ) } } } - private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { - // Refresh the video, maybe it was deleted - const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url) - - if (!video) { - logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url) - - await redundancyModel.destroy() - return - } - + private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) + // Redundancy strategy disabled, remove our redundancy instead of extending expiration + if (!redundancy) await removeVideoRedundancy(redundancyModel) + await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) } @@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { - const serverActor = await getServerActor() + private async createVideoRedundancies (data: CandidateToDuplicate) { + const video = await this.loadAndRefreshVideo(data.video.url) + + if (!video) { + logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) - for (const file of filesToDuplicate) { - const video = await this.loadAndRefreshVideo(file.Video.url) + return + } + for (const file of data.files) { const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) if (existingRedundancy) { - await this.extendsOrDeleteRedundancy(existingRedundancy) + await this.extendsRedundancy(existingRedundancy) continue } - if (!video) { - logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) + await this.createVideoFileRedundancy(data.redundancy, video, file) + } + + for (const streamingPlaylist of data.streamingPlaylists) { + const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) + if (existingRedundancy) { + await this.extendsRedundancy(existingRedundancy) continue } - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) + await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) + } + } - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) + private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { + file.Video = video - const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) + const serverActor = await getServerActor() - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) - await move(tmpPath, destPath) + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) - const createdModel = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), - url: getVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), - strategy: redundancy.strategy, - videoFileId: file.id, - actorId: serverActor.id - }) - createdModel.VideoFile = file + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) - await sendCreateCacheFile(serverActor, createdModel) + const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) - } + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) + await move(tmpPath, destPath) + + const createdModel = await VideoRedundancyModel.create({ + expiresOn: this.buildNewExpiration(redundancy.minLifetime), + url: getVideoCacheFileActivityPubUrl(file), + fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), + strategy: redundancy.strategy, + videoFileId: file.id, + actorId: serverActor.id + }) + + createdModel.VideoFile = file + + await sendCreateCacheFile(serverActor, video, createdModel) + + logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) + } + + private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { + playlist.Video = video + + const serverActor = await getServerActor() + + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) + + const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) + await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) + + const createdModel = await VideoRedundancyModel.create({ + expiresOn: this.buildNewExpiration(redundancy.minLifetime), + url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), + fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL), + strategy: redundancy.strategy, + videoStreamingPlaylistId: playlist.id, + actorId: serverActor.id + }) + + createdModel.VideoStreamingPlaylist = playlist + + await sendCreateCacheFile(serverActor, video, createdModel) + + logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) } private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { @@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await sendUpdateCacheFile(serverActor, redundancy) } - private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { - while (this.isTooHeavy(redundancy, filesToDuplicate)) { + private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { + while (this.isTooHeavy(candidateToDuplicate)) { + const redundancy = candidateToDuplicate.redundancy const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) if (!toDelete) return @@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { - const maxSize = redundancy.size + private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { + const maxSize = candidateToDuplicate.redundancy.size - const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) - const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) + const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) + const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) return totalWillDuplicate > maxSize } @@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private buildEntryLogId (object: VideoRedundancyModel) { - return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` + if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` + + return `${object.VideoStreamingPlaylist.playlistUrl}` } - private getTotalFileSizes (files: VideoFileModel[]) { + private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size - return files.reduce(fileReducer, 0) + return files.reduce(fileReducer, 0) * playlists.length } private async loadAndRefreshVideo (videoUrl: string) { diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 4460f46e4..608badfef 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -1,11 +1,14 @@ -import { CONFIG } from '../initializers' +import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' import { extname, join } from 'path' import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' -import { copy, remove, move, stat } from 'fs-extra' +import { copy, ensureDir, move, remove, stat } from 'fs-extra' import { logger } from '../helpers/logger' import { VideoResolution } from '../../shared/models/videos' import { VideoFileModel } from '../models/video/video-file' import { VideoModel } from '../models/video/video' +import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' +import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR @@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi const transcodeOptions = { inputPath: videoInputPath, - outputPath: videoTranscodedPath + outputPath: videoTranscodedPath, + resolution: inputVideoFile.resolution } // Could be very long! @@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi } } -async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { +async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const extname = '.mp4' @@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR size: 0, videoId: video.id }) - const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) + const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) const transcodeOptions = { inputPath: videoInputPath, outputPath: videoOutputPath, resolution, - isPortraitMode + isPortraitMode: isPortrait } await transcode(transcodeOptions) @@ -84,6 +88,38 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR video.VideoFiles.push(newVideoFile) } +async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { + const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid)) + + const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile())) + const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) + + const transcodeOptions = { + inputPath: videoInputPath, + outputPath, + resolution, + isPortraitMode, + generateHlsPlaylist: true + } + + await transcode(transcodeOptions) + + await updateMasterHLSPlaylist(video) + await updateSha256Segments(video) + + const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) + + await VideoStreamingPlaylistModel.upsert({ + videoId: video.id, + playlistUrl, + segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), + p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles), + + type: VideoStreamingPlaylistType.HLS + }) +} + async function importVideoFile (video: VideoModel, inputFilePath: string) { const { videoFileResolution } = await getVideoFileResolution(inputFilePath) const { size } = await stat(inputFilePath) @@ -125,6 +161,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) { } export { + generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile, importVideoFile -- cgit v1.2.3 From 4c280004ce62bf11ddb091854c28f1e1d54a54d6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 7 Feb 2019 15:08:19 +0100 Subject: Use a single file instead of segments for HLS --- server/lib/activitypub/actor.ts | 4 +- server/lib/hls.ts | 136 ++++++++++++++++++++++++++++------------ server/lib/video-transcoding.ts | 5 +- 3 files changed, 101 insertions(+), 44 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 8215840da..a3f379b76 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -355,10 +355,10 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe logger.info('Fetching remote actor %s.', actorUrl) - const requestResult = await doRequest(options) + const requestResult = await doRequest(options) normalizeActor(requestResult.body) - const actorJSON: ActivityPubActor = requestResult.body + const actorJSON = requestResult.body if (isActorObjectValid(actorJSON) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON }) return { result: undefined, statusCode: requestResult.response.statusCode } diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 10db6c3c3..3575981f4 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,13 +1,14 @@ import { VideoModel } from '../models/video/video' -import { basename, dirname, join } from 'path' -import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' -import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' +import { basename, join, dirname } from 'path' +import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' +import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' import { getVideoFileSize } from '../helpers/ffmpeg-utils' import { sha256 } from '../helpers/core-utils' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import HLSDownloader from 'hlsdownloader' import { logger } from '../helpers/logger' -import { parse } from 'url' +import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' +import { generateRandomString } from '../helpers/utils' +import { flatten, uniq } from 'lodash' async function updateMasterHLSPlaylist (video: VideoModel) { const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) @@ -37,66 +38,119 @@ async function updateMasterHLSPlaylist (video: VideoModel) { } async function updateSha256Segments (video: VideoModel) { - const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) - const files = await readdir(directory) - const json: { [filename: string]: string} = {} + const json: { [filename: string]: { [range: string]: string } } = {} + + const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + + // For all the resolutions available for this video + for (const file of video.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} + + const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) + const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) - for (const file of files) { - if (file.endsWith('.ts') === false) continue + // Maybe the playlist is not generated for this resolution yet + if (!await pathExists(playlistPath)) continue - const buffer = await readFile(join(directory, file)) - const filename = basename(file) + const playlistContent = await readFile(playlistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) - json[filename] = sha256(buffer) + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) + + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) + + const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) + json[videoFilename] = rangeHashes } - const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) await outputJSON(outputPath, json) } -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { - let timer +function getRangesFromPlaylist (playlistContent: string) { + const ranges: { offset: number, length: number }[] = [] + const lines = playlistContent.split('\n') + const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ - logger.info('Importing HLS playlist %s', playlistUrl) + for (const line of lines) { + const captured = regex.exec(line) - const params = { - playlistURL: playlistUrl, - destination: CONFIG.STORAGE.TMP_DIR + if (captured) { + ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) + } } - const downloader = new HLSDownloader(params) - - const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname)) - return new Promise(async (res, rej) => { - downloader.startDownload(err => { - clearTimeout(timer) + return ranges +} - if (err) { - deleteTmpDirectory(hlsDestinationDir) +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { + let timer - return rej(err) - } + logger.info('Importing HLS playlist %s', playlistUrl) - move(hlsDestinationDir, destinationDir, { overwrite: true }) - .then(() => res()) - .catch(err => { - deleteTmpDirectory(hlsDestinationDir) + return new Promise(async (res, rej) => { + const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) - return rej(err) - }) - }) + await ensureDir(tmpDirectory) timer = setTimeout(() => { - deleteTmpDirectory(hlsDestinationDir) + deleteTmpDirectory(tmpDirectory) return rej(new Error('HLS download timeout.')) }, timeout) - function deleteTmpDirectory (directory: string) { - remove(directory) - .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + try { + // Fetch master playlist + const subPlaylistUrls = await fetchUniqUrls(playlistUrl) + + const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) + const fileUrls = uniq(flatten(await Promise.all(subRequests))) + + logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) + + for (const fileUrl of fileUrls) { + const destPath = join(tmpDirectory, basename(fileUrl)) + + await doRequestAndSaveToFile({ uri: fileUrl }, destPath) + } + + clearTimeout(timer) + + await move(tmpDirectory, destinationDir, { overwrite: true }) + + return res() + } catch (err) { + deleteTmpDirectory(tmpDirectory) + + return rej(err) } }) + + function deleteTmpDirectory (directory: string) { + remove(directory) + .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + } + + async function fetchUniqUrls (playlistUrl: string) { + const { body } = await doRequest({ uri: playlistUrl }) + + if (!body) return [] + + const urls = body.split('\n') + .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) + .map(url => { + if (url.startsWith('http://') || url.startsWith('https://')) return url + + return `${dirname(playlistUrl)}/${url}` + }) + + return uniq(urls) + } } // --------------------------------------------------------------------------- diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 608badfef..086b860a2 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -100,7 +100,10 @@ async function generateHlsPlaylist (video: VideoModel, resolution: VideoResoluti outputPath, resolution, isPortraitMode, - generateHlsPlaylist: true + + hlsPlaylist: { + videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) + } } await transcode(transcodeOptions) -- cgit v1.2.3 From 597a9266d426aa04c2f229168e4285a76bea2c12 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 7 Feb 2019 15:56:17 +0100 Subject: Add player mode in watch/embed urls --- server/lib/job-queue/handlers/video-file.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/lib') diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 7119ce0ca..04983155c 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -172,7 +172,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: Video // don't notify prior to scheduled video update if (!videoDatabase.ScheduleVideoUpdate) { - if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } -- cgit v1.2.3 From 328c78bc4a570a9aceaaa1a2124bacd4a0e8d295 Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Sat, 6 Oct 2018 13:54:00 +0200 Subject: allow administration to change/reset a user's password --- server/lib/emailer.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'server/lib') diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index f384a254e..7681164b3 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -101,6 +101,22 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } + addForceResetPasswordEmailJob (to: string, resetPasswordUrl: string) { + const text = `Hi dear user,\n\n` + + `Your password has been reset on ${CONFIG.WEBSERVER.HOST}! ` + + `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to: [ to ], + subject: 'Reset of your PeerTube password', + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + addNewFollowNotification (to: string[], actorFollow: ActorFollowModel, followType: 'account' | 'channel') { const followerName = actorFollow.ActorFollower.Account.getDisplayName() const followingName = (actorFollow.ActorFollowing.VideoChannel || actorFollow.ActorFollowing.Account).getDisplayName() -- cgit v1.2.3 From b426edd4854adc6e65844d8c54b8998e792b5778 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 11 Feb 2019 09:30:29 +0100 Subject: Cleanup reset user password by admin And add some tests --- server/lib/emailer.ts | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) (limited to 'server/lib') diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 7681164b3..672414cc0 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -101,22 +101,6 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - addForceResetPasswordEmailJob (to: string, resetPasswordUrl: string) { - const text = `Hi dear user,\n\n` + - `Your password has been reset on ${CONFIG.WEBSERVER.HOST}! ` + - `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + - `Cheers,\n` + - `PeerTube.` - - const emailPayload: EmailPayload = { - to: [ to ], - subject: 'Reset of your PeerTube password', - text - } - - return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) - } - addNewFollowNotification (to: string[], actorFollow: ActorFollowModel, followType: 'account' | 'channel') { const followerName = actorFollow.ActorFollower.Account.getDisplayName() const followingName = (actorFollow.ActorFollowing.VideoChannel || actorFollow.ActorFollowing.Account).getDisplayName() @@ -312,9 +296,9 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { + addPasswordResetEmailJob (to: string, resetPasswordUrl: string) { const text = `Hi dear user,\n\n` + - `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + + `A reset password procedure for your account ${to} has been requested on ${CONFIG.WEBSERVER.HOST} ` + `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + `If you are not the person who initiated this request, please ignore this email.\n\n` + `Cheers,\n` + -- cgit v1.2.3