From 156c50af3085468a47b8ae73fe8cfcae46b42398 Mon Sep 17 00:00:00 2001 From: Lucas Declercq Date: Sat, 6 Oct 2018 19:17:21 +0200 Subject: Add downloadingEnabled property to video model --- server/controllers/api/videos/import.ts | 1 + server/controllers/api/videos/index.ts | 2 ++ server/helpers/activitypub.ts | 1 + server/helpers/audit-logger.ts | 3 ++- .../custom-validators/activitypub/videos.ts | 1 + .../migrations/0280-video-downloading-enabled.ts | 27 ++++++++++++++++++++++ server/lib/activitypub/videos.ts | 2 ++ server/middlewares/validators/videos/videos.ts | 4 ++++ server/models/video/video-format-utils.ts | 2 ++ server/models/video/video.ts | 4 ++++ server/tests/api/check-params/video-imports.ts | 1 + server/tests/api/check-params/videos.ts | 2 ++ server/tests/api/server/follows.ts | 1 + server/tests/api/server/handle-down.ts | 1 + server/tests/api/videos/multiple-servers.ts | 6 +++++ server/tests/api/videos/single-server.ts | 3 +++ server/tests/utils/videos/videos.ts | 6 +++++ server/tools/peertube-import-videos.ts | 1 + server/tools/peertube-upload.ts | 2 ++ 19 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 server/initializers/migrations/0280-video-downloading-enabled.ts (limited to 'server') diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 398fd5a7f..a5cddba89 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -171,6 +171,7 @@ function buildVideo (channelId: number, body: VideoImportCreate, importData: You licence: body.licence || importData.licence, language: body.language || undefined, commentsEnabled: body.commentsEnabled || true, + downloadingEnabled: body.downloadingEnabled || true, waitTranscoding: body.waitTranscoding || false, state: VideoState.TO_IMPORT, nsfw: body.nsfw || importData.nsfw || false, diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 6a73e13d0..ec25006e8 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -179,6 +179,7 @@ async function addVideo (req: express.Request, res: express.Response) { licence: videoInfo.licence, language: videoInfo.language, commentsEnabled: videoInfo.commentsEnabled || false, + downloadingEnabled: videoInfo.downloadingEnabled || false, waitTranscoding: videoInfo.waitTranscoding || false, state: CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED, nsfw: videoInfo.nsfw || false, @@ -322,6 +323,7 @@ async function updateVideo (req: express.Request, res: express.Response) { if (videoInfoToUpdate.support !== undefined) videoInstance.set('support', videoInfoToUpdate.support) if (videoInfoToUpdate.description !== undefined) videoInstance.set('description', videoInfoToUpdate.description) if (videoInfoToUpdate.commentsEnabled !== undefined) videoInstance.set('commentsEnabled', videoInfoToUpdate.commentsEnabled) + if (videoInfoToUpdate.downloadingEnabled !== undefined) videoInstance.set('downloadingEnabled', videoInfoToUpdate.downloadingEnabled) if (videoInfoToUpdate.privacy !== undefined) { const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10) videoInstance.set('privacy', newPrivacy) diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 1304c7559..7f903e486 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -28,6 +28,7 @@ function activityPubContextify (data: T) { size: 'schema:Number', fps: 'schema:Number', commentsEnabled: 'schema:Boolean', + downloadingEnabled: 'schema:Boolean', waitTranscoding: 'schema:Boolean', expires: 'schema:expires', support: 'schema:Text', diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index 00311fce1..d2c9aee01 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts @@ -117,7 +117,8 @@ const videoKeysToKeep = [ 'channel-uuid', 'channel-name', 'support', - 'commentsEnabled' + 'commentsEnabled', + 'downloadingEnabled' ] class VideoAuditView extends EntityAuditView { constructor (private video: VideoDetails) { diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index f88d26561..34e4cdff9 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -67,6 +67,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { isVideoViewsValid(video.views) && isBooleanValid(video.sensitive) && isBooleanValid(video.commentsEnabled) && + isBooleanValid(video.downloadingEnabled) && isDateValid(video.published) && isDateValid(video.updated) && (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && diff --git a/server/initializers/migrations/0280-video-downloading-enabled.ts b/server/initializers/migrations/0280-video-downloading-enabled.ts new file mode 100644 index 000000000..c0700108c --- /dev/null +++ b/server/initializers/migrations/0280-video-downloading-enabled.ts @@ -0,0 +1,27 @@ +import * as Sequelize from 'sequelize' +import { Migration } from '../../models/migrations' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize +}): Promise { + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: true + } as Migration.Boolean + await utils.queryInterface.addColumn('video', 'downloadingEnabled', data) + + data.defaultValue = null + return utils.queryInterface.changeColumn('video', 'downloadingEnabled', data) +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 54cea542f..dd02141ee 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -230,6 +230,7 @@ async function updateVideoFromAP (options: { options.video.set('support', videoData.support) options.video.set('nsfw', videoData.nsfw) options.video.set('commentsEnabled', videoData.commentsEnabled) + options.video.set('downloadingEnabled', videoData.downloadingEnabled) options.video.set('waitTranscoding', videoData.waitTranscoding) options.video.set('state', videoData.state) options.video.set('duration', videoData.duration) @@ -441,6 +442,7 @@ async function videoActivityObjectToDBAttributes ( support, nsfw: videoObject.sensitive, commentsEnabled: videoObject.commentsEnabled, + downloadingEnabled: videoObject.downloadingEnabled, waitTranscoding: videoObject.waitTranscoding, state: videoObject.state, channelId: videoChannel.id, diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index d6b8aa725..bdba87496 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -349,6 +349,10 @@ function getCommonVideoAttributes () { .optional() .toBoolean() .custom(isBooleanValid).withMessage('Should have comments enabled boolean'), + body('downloadingEnabled') + .optional() + .toBoolean() + .custom(isBooleanValid).withMessage('Should have downloading enabled boolean'), body('scheduleUpdate') .optional() diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index 905e84449..0b0da4181 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -128,6 +128,7 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { account: video.VideoChannel.Account.toFormattedJSON(), tags, commentsEnabled: video.commentsEnabled, + downloadingEnabled: video.downloadingEnabled, waitTranscoding: video.waitTranscoding, state: { id: video.state, @@ -259,6 +260,7 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { waitTranscoding: video.waitTranscoding, state: video.state, commentsEnabled: video.commentsEnabled, + downloadingEnabled: video.downloadingEnabled, published: video.publishedAt.toISOString(), updated: video.updatedAt.toISOString(), mediaType: 'text/markdown', diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 46d823240..a2fe53fb9 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -598,6 +598,10 @@ export class VideoModel extends Model { @Column commentsEnabled: boolean + @AllowNull(false) + @Column + downloadingEnabled: boolean + @AllowNull(false) @Column waitTranscoding: boolean diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts index 44645b0e2..8dd87b8ae 100644 --- a/server/tests/api/check-params/video-imports.ts +++ b/server/tests/api/check-params/video-imports.ts @@ -84,6 +84,7 @@ describe('Test video imports API validator', function () { language: 'pt', nsfw: false, commentsEnabled: true, + downloadingEnabled: true, waitTranscoding: true, description: 'my super description', support: 'my super support text', diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index 904d22870..c5740087c 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -175,6 +175,7 @@ describe('Test videos API validator', function () { language: 'pt', nsfw: false, commentsEnabled: true, + downloadingEnabled: true, waitTranscoding: true, description: 'my super description', support: 'my super support text', @@ -419,6 +420,7 @@ describe('Test videos API validator', function () { language: 'pt', nsfw: false, commentsEnabled: false, + downloadingEnabled: false, description: 'my super description', privacy: VideoPrivacy.PUBLIC, tags: [ 'tag1', 'tag2' ] diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts index 310c291bf..5cad1d09d 100644 --- a/server/tests/api/server/follows.ts +++ b/server/tests/api/server/follows.ts @@ -305,6 +305,7 @@ describe('Test follows', function () { }, isLocal, commentsEnabled: true, + downloadingEnabled: true, duration: 5, tags: [ 'tag1', 'tag2', 'tag3' ], privacy: VideoPrivacy.PUBLIC, diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index ed15c8090..971de4607 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts @@ -70,6 +70,7 @@ describe('Test handle downs', function () { tags: [ 'tag1p1', 'tag2p1' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, + downloadingEnabled: true, channel: { name: 'root_channel', displayName: 'Main root channel', diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 4553ee855..83e391ccd 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -127,6 +127,7 @@ describe('Test multiple servers', function () { tags: [ 'tag1p1', 'tag2p1' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, + downloadingEnabled: true, channel: { displayName: 'my channel', name: 'super_channel_name', @@ -198,6 +199,7 @@ describe('Test multiple servers', function () { }, isLocal, commentsEnabled: true, + downloadingEnabled: true, duration: 5, tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ], privacy: VideoPrivacy.PUBLIC, @@ -306,6 +308,7 @@ describe('Test multiple servers', function () { isLocal, duration: 5, commentsEnabled: true, + downloadingEnabled: true, tags: [ 'tag1p3' ], privacy: VideoPrivacy.PUBLIC, channel: { @@ -337,6 +340,7 @@ describe('Test multiple servers', function () { host: 'localhost:9003' }, commentsEnabled: true, + downloadingEnabled: true, isLocal, duration: 5, tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ], @@ -654,6 +658,7 @@ describe('Test multiple servers', function () { isLocal, duration: 5, commentsEnabled: true, + downloadingEnabled: true, tags: [ 'tag_up_1', 'tag_up_2' ], privacy: VideoPrivacy.PUBLIC, channel: { @@ -975,6 +980,7 @@ describe('Test multiple servers', function () { isLocal, duration: 5, commentsEnabled: false, + downloadingEnabled: false, tags: [ ], privacy: VideoPrivacy.PUBLIC, channel: { diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index e3d62b7a0..8995a8525 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -55,6 +55,7 @@ describe('Test a single server', function () { tags: [ 'tag1', 'tag2', 'tag3' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, + downloadingEnabled: true, channel: { displayName: 'Main root channel', name: 'root_channel', @@ -87,6 +88,7 @@ describe('Test a single server', function () { privacy: VideoPrivacy.PUBLIC, duration: 5, commentsEnabled: false, + downloadingEnabled: false, channel: { name: 'root_channel', displayName: 'Main root channel', @@ -356,6 +358,7 @@ describe('Test a single server', function () { nsfw: false, description: 'my super description updated', commentsEnabled: false, + downloadingEnabled: false, tags: [ 'tagup1', 'tagup2' ] } await updateVideo(server.url, server.accessToken, videoId, attributes) diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts index 87c385f38..a7fd4c8a6 100644 --- a/server/tests/utils/videos/videos.ts +++ b/server/tests/utils/videos/videos.ts @@ -27,6 +27,7 @@ type VideoAttributes = { language?: string nsfw?: boolean commentsEnabled?: boolean + downloadingEnabled?: boolean waitTranscoding?: boolean description?: string tags?: string[] @@ -310,6 +311,7 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg tags: [ 'tag' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, + downloadingEnabled: true, fixture: 'video_short.webm' }, videoAttributesArg) @@ -320,6 +322,7 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg .field('name', attributes.name) .field('nsfw', JSON.stringify(attributes.nsfw)) .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled)) + .field('downloadingEnabled', JSON.stringify(attributes.downloadingEnabled)) .field('waitTranscoding', JSON.stringify(attributes.waitTranscoding)) .field('privacy', attributes.privacy.toString()) .field('channelId', attributes.channelId) @@ -370,6 +373,7 @@ function updateVideo (url: string, accessToken: string, id: number | string, att if (attributes.language) body['language'] = attributes.language if (attributes.nsfw !== undefined) body['nsfw'] = JSON.stringify(attributes.nsfw) if (attributes.commentsEnabled !== undefined) body['commentsEnabled'] = JSON.stringify(attributes.commentsEnabled) + if (attributes.downloadingEnabled !== undefined) body['downloadingEnabled'] = JSON.stringify(attributes.downloadingEnabled) if (attributes.description) body['description'] = attributes.description if (attributes.tags) body['tags'] = attributes.tags if (attributes.privacy) body['privacy'] = attributes.privacy @@ -435,6 +439,7 @@ async function completeVideoCheck ( language: string nsfw: boolean commentsEnabled: boolean + downloadingEnabled: boolean description: string publishedAt?: string support: string @@ -509,6 +514,7 @@ async function completeVideoCheck ( expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled) + expect(videoDetails.downloadingEnabled).to.equal(attributes.downloadingEnabled) for (const attributeFile of attributes.files) { const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution) diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts index 13090a028..675c621df 100644 --- a/server/tools/peertube-import-videos.ts +++ b/server/tools/peertube-import-videos.ts @@ -212,6 +212,7 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: st nsfw: isNSFW(videoInfo), waitTranscoding: true, commentsEnabled: true, + downloadingEnabled: true, description: videoInfo.description || undefined, support: undefined, tags, diff --git a/server/tools/peertube-upload.ts b/server/tools/peertube-upload.ts index 6248fb47d..e7b885a38 100644 --- a/server/tools/peertube-upload.ts +++ b/server/tools/peertube-upload.ts @@ -30,6 +30,7 @@ if (!program['tags']) program['tags'] = [] if (!program['nsfw']) program['nsfw'] = false if (!program['privacy']) program['privacy'] = VideoPrivacy.PUBLIC if (!program['commentsEnabled']) program['commentsEnabled'] = false +if (!program['downloadingEnabled']) program['downloadingEnabled'] = false getSettings() .then(settings => { @@ -116,6 +117,7 @@ async function run () { description: program['videoDescription'], tags: program['tags'], commentsEnabled: program['commentsEnabled'], + downloadingEnabled: program['downloadingEnabled'], fixture: program['file'], thumbnailfile: program['thumbnail'], previewfile: program['preview'], -- cgit v1.2.3 From 4ffdcfc63b8c804a0aea20609544c859ab57318b Mon Sep 17 00:00:00 2001 From: Lucas Declercq Date: Mon, 8 Oct 2018 14:42:55 +0200 Subject: Fix some defaults values + indentation --- server/controllers/api/videos/index.ts | 2 +- server/helpers/custom-validators/activitypub/videos.ts | 1 + server/tools/peertube-upload.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'server') diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index ec25006e8..4b6d1b847 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -179,7 +179,7 @@ async function addVideo (req: express.Request, res: express.Response) { licence: videoInfo.licence, language: videoInfo.language, commentsEnabled: videoInfo.commentsEnabled || false, - downloadingEnabled: videoInfo.downloadingEnabled || false, + downloadingEnabled: videoInfo.downloadingEnabled || true, waitTranscoding: videoInfo.waitTranscoding || false, state: CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED, nsfw: videoInfo.nsfw || false, diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 34e4cdff9..59964f91a 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -56,6 +56,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { // Default attributes if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false + if (!isBooleanValid(video.downloadingEnabled)) video.downloadingEnabled = true return isActivityPubUrlValid(video.id) && isVideoNameValid(video.name) && diff --git a/server/tools/peertube-upload.ts b/server/tools/peertube-upload.ts index e7b885a38..e2ba4bdbc 100644 --- a/server/tools/peertube-upload.ts +++ b/server/tools/peertube-upload.ts @@ -30,7 +30,7 @@ if (!program['tags']) program['tags'] = [] if (!program['nsfw']) program['nsfw'] = false if (!program['privacy']) program['privacy'] = VideoPrivacy.PUBLIC if (!program['commentsEnabled']) program['commentsEnabled'] = false -if (!program['downloadingEnabled']) program['downloadingEnabled'] = false +if (!program['downloadingEnabled']) program['downloadingEnabled'] = true getSettings() .then(settings => { -- cgit v1.2.3 From 7f2cfe3a792856f7de6f1d13688aa3d06ec1bf70 Mon Sep 17 00:00:00 2001 From: Lucas Declercq Date: Mon, 8 Oct 2018 14:45:22 +0200 Subject: Rename downloadingEnabled property to downloadEnabled --- server/controllers/api/videos/import.ts | 2 +- server/controllers/api/videos/index.ts | 4 ++-- server/helpers/activitypub.ts | 2 +- server/helpers/audit-logger.ts | 2 +- server/helpers/custom-validators/activitypub/videos.ts | 4 ++-- .../migrations/0280-video-downloading-enabled.ts | 4 ++-- server/lib/activitypub/videos.ts | 4 ++-- server/middlewares/validators/videos/videos.ts | 2 +- server/models/video/video-format-utils.ts | 4 ++-- server/models/video/video.ts | 2 +- server/tests/api/check-params/video-imports.ts | 2 +- server/tests/api/check-params/videos.ts | 4 ++-- server/tests/api/server/follows.ts | 2 +- server/tests/api/server/handle-down.ts | 2 +- server/tests/api/videos/multiple-servers.ts | 12 ++++++------ server/tests/api/videos/single-server.ts | 6 +++--- server/tests/utils/videos/videos.ts | 12 ++++++------ server/tools/peertube-import-videos.ts | 2 +- server/tools/peertube-upload.ts | 4 ++-- 19 files changed, 38 insertions(+), 38 deletions(-) (limited to 'server') diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index a5cddba89..9e51e2000 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -171,7 +171,7 @@ function buildVideo (channelId: number, body: VideoImportCreate, importData: You licence: body.licence || importData.licence, language: body.language || undefined, commentsEnabled: body.commentsEnabled || true, - downloadingEnabled: body.downloadingEnabled || true, + downloadEnabled: body.downloadEnabled || true, waitTranscoding: body.waitTranscoding || false, state: VideoState.TO_IMPORT, nsfw: body.nsfw || importData.nsfw || false, diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 4b6d1b847..7d55f06b6 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -179,7 +179,7 @@ async function addVideo (req: express.Request, res: express.Response) { licence: videoInfo.licence, language: videoInfo.language, commentsEnabled: videoInfo.commentsEnabled || false, - downloadingEnabled: videoInfo.downloadingEnabled || true, + downloadEnabled: videoInfo.downloadEnabled || true, waitTranscoding: videoInfo.waitTranscoding || false, state: CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED, nsfw: videoInfo.nsfw || false, @@ -323,7 +323,7 @@ async function updateVideo (req: express.Request, res: express.Response) { if (videoInfoToUpdate.support !== undefined) videoInstance.set('support', videoInfoToUpdate.support) if (videoInfoToUpdate.description !== undefined) videoInstance.set('description', videoInfoToUpdate.description) if (videoInfoToUpdate.commentsEnabled !== undefined) videoInstance.set('commentsEnabled', videoInfoToUpdate.commentsEnabled) - if (videoInfoToUpdate.downloadingEnabled !== undefined) videoInstance.set('downloadingEnabled', videoInfoToUpdate.downloadingEnabled) + if (videoInfoToUpdate.downloadEnabled !== undefined) videoInstance.set('downloadEnabled', videoInfoToUpdate.downloadEnabled) if (videoInfoToUpdate.privacy !== undefined) { const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10) videoInstance.set('privacy', newPrivacy) diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 7f903e486..2469b37b1 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -28,7 +28,7 @@ function activityPubContextify (data: T) { size: 'schema:Number', fps: 'schema:Number', commentsEnabled: 'schema:Boolean', - downloadingEnabled: 'schema:Boolean', + downloadEnabled: 'schema:Boolean', waitTranscoding: 'schema:Boolean', expires: 'schema:expires', support: 'schema:Text', diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index d2c9aee01..a121f0b8a 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts @@ -118,7 +118,7 @@ const videoKeysToKeep = [ 'channel-name', 'support', 'commentsEnabled', - 'downloadingEnabled' + 'downloadEnabled' ] class VideoAuditView extends EntityAuditView { constructor (private video: VideoDetails) { diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 59964f91a..5015c59dd 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -56,7 +56,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { // Default attributes if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false - if (!isBooleanValid(video.downloadingEnabled)) video.downloadingEnabled = true + if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true return isActivityPubUrlValid(video.id) && isVideoNameValid(video.name) && @@ -68,7 +68,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { isVideoViewsValid(video.views) && isBooleanValid(video.sensitive) && isBooleanValid(video.commentsEnabled) && - isBooleanValid(video.downloadingEnabled) && + isBooleanValid(video.downloadEnabled) && isDateValid(video.published) && isDateValid(video.updated) && (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && diff --git a/server/initializers/migrations/0280-video-downloading-enabled.ts b/server/initializers/migrations/0280-video-downloading-enabled.ts index c0700108c..e79466447 100644 --- a/server/initializers/migrations/0280-video-downloading-enabled.ts +++ b/server/initializers/migrations/0280-video-downloading-enabled.ts @@ -11,10 +11,10 @@ async function up (utils: { allowNull: false, defaultValue: true } as Migration.Boolean - await utils.queryInterface.addColumn('video', 'downloadingEnabled', data) + await utils.queryInterface.addColumn('video', 'downloadEnabled', data) data.defaultValue = null - return utils.queryInterface.changeColumn('video', 'downloadingEnabled', data) + return utils.queryInterface.changeColumn('video', 'downloadEnabled', data) } function down (options) { diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index dd02141ee..8521572a1 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -230,7 +230,7 @@ async function updateVideoFromAP (options: { options.video.set('support', videoData.support) options.video.set('nsfw', videoData.nsfw) options.video.set('commentsEnabled', videoData.commentsEnabled) - options.video.set('downloadingEnabled', videoData.downloadingEnabled) + options.video.set('downloadEnabled', videoData.downloadEnabled) options.video.set('waitTranscoding', videoData.waitTranscoding) options.video.set('state', videoData.state) options.video.set('duration', videoData.duration) @@ -442,7 +442,7 @@ async function videoActivityObjectToDBAttributes ( support, nsfw: videoObject.sensitive, commentsEnabled: videoObject.commentsEnabled, - downloadingEnabled: videoObject.downloadingEnabled, + downloadEnabled: videoObject.downloadEnabled, waitTranscoding: videoObject.waitTranscoding, state: videoObject.state, channelId: videoChannel.id, diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index bdba87496..27e8a7449 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -349,7 +349,7 @@ function getCommonVideoAttributes () { .optional() .toBoolean() .custom(isBooleanValid).withMessage('Should have comments enabled boolean'), - body('downloadingEnabled') + body('downloadEnabled') .optional() .toBoolean() .custom(isBooleanValid).withMessage('Should have downloading enabled boolean'), diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index 0b0da4181..e7bff2ed7 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -128,7 +128,7 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { account: video.VideoChannel.Account.toFormattedJSON(), tags, commentsEnabled: video.commentsEnabled, - downloadingEnabled: video.downloadingEnabled, + downloadEnabled: video.downloadEnabled, waitTranscoding: video.waitTranscoding, state: { id: video.state, @@ -260,7 +260,7 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { waitTranscoding: video.waitTranscoding, state: video.state, commentsEnabled: video.commentsEnabled, - downloadingEnabled: video.downloadingEnabled, + downloadEnabled: video.downloadEnabled, published: video.publishedAt.toISOString(), updated: video.updatedAt.toISOString(), mediaType: 'text/markdown', diff --git a/server/models/video/video.ts b/server/models/video/video.ts index a2fe53fb9..a9baaf1da 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -600,7 +600,7 @@ export class VideoModel extends Model { @AllowNull(false) @Column - downloadingEnabled: boolean + downloadEnabled: boolean @AllowNull(false) @Column diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts index 8dd87b8ae..1ffb81a38 100644 --- a/server/tests/api/check-params/video-imports.ts +++ b/server/tests/api/check-params/video-imports.ts @@ -84,7 +84,7 @@ describe('Test video imports API validator', function () { language: 'pt', nsfw: false, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, waitTranscoding: true, description: 'my super description', support: 'my super support text', diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index c5740087c..bc28e2422 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -175,7 +175,7 @@ describe('Test videos API validator', function () { language: 'pt', nsfw: false, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, waitTranscoding: true, description: 'my super description', support: 'my super support text', @@ -420,7 +420,7 @@ describe('Test videos API validator', function () { language: 'pt', nsfw: false, commentsEnabled: false, - downloadingEnabled: false, + downloadEnabled: false, description: 'my super description', privacy: VideoPrivacy.PUBLIC, tags: [ 'tag1', 'tag2' ] diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts index 5cad1d09d..e8914a9d4 100644 --- a/server/tests/api/server/follows.ts +++ b/server/tests/api/server/follows.ts @@ -305,7 +305,7 @@ describe('Test follows', function () { }, isLocal, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, duration: 5, tags: [ 'tag1', 'tag2', 'tag3' ], privacy: VideoPrivacy.PUBLIC, diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index 971de4607..b0a3d029a 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts @@ -70,7 +70,7 @@ describe('Test handle downs', function () { tags: [ 'tag1p1', 'tag2p1' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, channel: { name: 'root_channel', displayName: 'Main root channel', diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 83e391ccd..256be5d1c 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -127,7 +127,7 @@ describe('Test multiple servers', function () { tags: [ 'tag1p1', 'tag2p1' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, channel: { displayName: 'my channel', name: 'super_channel_name', @@ -199,7 +199,7 @@ describe('Test multiple servers', function () { }, isLocal, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, duration: 5, tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ], privacy: VideoPrivacy.PUBLIC, @@ -308,7 +308,7 @@ describe('Test multiple servers', function () { isLocal, duration: 5, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, tags: [ 'tag1p3' ], privacy: VideoPrivacy.PUBLIC, channel: { @@ -340,7 +340,7 @@ describe('Test multiple servers', function () { host: 'localhost:9003' }, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, isLocal, duration: 5, tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ], @@ -658,7 +658,7 @@ describe('Test multiple servers', function () { isLocal, duration: 5, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, tags: [ 'tag_up_1', 'tag_up_2' ], privacy: VideoPrivacy.PUBLIC, channel: { @@ -980,7 +980,7 @@ describe('Test multiple servers', function () { isLocal, duration: 5, commentsEnabled: false, - downloadingEnabled: false, + downloadEnabled: false, tags: [ ], privacy: VideoPrivacy.PUBLIC, channel: { diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index 8995a8525..92d42eb80 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -55,7 +55,7 @@ describe('Test a single server', function () { tags: [ 'tag1', 'tag2', 'tag3' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, channel: { displayName: 'Main root channel', name: 'root_channel', @@ -88,7 +88,7 @@ describe('Test a single server', function () { privacy: VideoPrivacy.PUBLIC, duration: 5, commentsEnabled: false, - downloadingEnabled: false, + downloadEnabled: false, channel: { name: 'root_channel', displayName: 'Main root channel', @@ -358,7 +358,7 @@ describe('Test a single server', function () { nsfw: false, description: 'my super description updated', commentsEnabled: false, - downloadingEnabled: false, + downloadEnabled: false, tags: [ 'tagup1', 'tagup2' ] } await updateVideo(server.url, server.accessToken, videoId, attributes) diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts index a7fd4c8a6..bc878b039 100644 --- a/server/tests/utils/videos/videos.ts +++ b/server/tests/utils/videos/videos.ts @@ -27,7 +27,7 @@ type VideoAttributes = { language?: string nsfw?: boolean commentsEnabled?: boolean - downloadingEnabled?: boolean + downloadEnabled?: boolean waitTranscoding?: boolean description?: string tags?: string[] @@ -311,7 +311,7 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg tags: [ 'tag' ], privacy: VideoPrivacy.PUBLIC, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, fixture: 'video_short.webm' }, videoAttributesArg) @@ -322,7 +322,7 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg .field('name', attributes.name) .field('nsfw', JSON.stringify(attributes.nsfw)) .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled)) - .field('downloadingEnabled', JSON.stringify(attributes.downloadingEnabled)) + .field('downloadEnabled', JSON.stringify(attributes.downloadEnabled)) .field('waitTranscoding', JSON.stringify(attributes.waitTranscoding)) .field('privacy', attributes.privacy.toString()) .field('channelId', attributes.channelId) @@ -373,7 +373,7 @@ function updateVideo (url: string, accessToken: string, id: number | string, att if (attributes.language) body['language'] = attributes.language if (attributes.nsfw !== undefined) body['nsfw'] = JSON.stringify(attributes.nsfw) if (attributes.commentsEnabled !== undefined) body['commentsEnabled'] = JSON.stringify(attributes.commentsEnabled) - if (attributes.downloadingEnabled !== undefined) body['downloadingEnabled'] = JSON.stringify(attributes.downloadingEnabled) + if (attributes.downloadEnabled !== undefined) body['downloadEnabled'] = JSON.stringify(attributes.downloadEnabled) if (attributes.description) body['description'] = attributes.description if (attributes.tags) body['tags'] = attributes.tags if (attributes.privacy) body['privacy'] = attributes.privacy @@ -439,7 +439,7 @@ async function completeVideoCheck ( language: string nsfw: boolean commentsEnabled: boolean - downloadingEnabled: boolean + downloadEnabled: boolean description: string publishedAt?: string support: string @@ -514,7 +514,7 @@ async function completeVideoCheck ( expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled) - expect(videoDetails.downloadingEnabled).to.equal(attributes.downloadingEnabled) + expect(videoDetails.downloadEnabled).to.equal(attributes.downloadEnabled) for (const attributeFile of attributes.files) { const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution) diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts index 675c621df..15f517cab 100644 --- a/server/tools/peertube-import-videos.ts +++ b/server/tools/peertube-import-videos.ts @@ -212,7 +212,7 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: st nsfw: isNSFW(videoInfo), waitTranscoding: true, commentsEnabled: true, - downloadingEnabled: true, + downloadEnabled: true, description: videoInfo.description || undefined, support: undefined, tags, diff --git a/server/tools/peertube-upload.ts b/server/tools/peertube-upload.ts index e2ba4bdbc..b17bc4288 100644 --- a/server/tools/peertube-upload.ts +++ b/server/tools/peertube-upload.ts @@ -30,7 +30,7 @@ if (!program['tags']) program['tags'] = [] if (!program['nsfw']) program['nsfw'] = false if (!program['privacy']) program['privacy'] = VideoPrivacy.PUBLIC if (!program['commentsEnabled']) program['commentsEnabled'] = false -if (!program['downloadingEnabled']) program['downloadingEnabled'] = true +if (!program['downloadEnabled']) program['downloadEnabled'] = true getSettings() .then(settings => { @@ -117,7 +117,7 @@ async function run () { description: program['videoDescription'], tags: program['tags'], commentsEnabled: program['commentsEnabled'], - downloadingEnabled: program['downloadingEnabled'], + downloadEnabled: program['downloadEnabled'], fixture: program['file'], thumbnailfile: program['thumbnail'], previewfile: program['preview'], -- cgit v1.2.3 From 5abb9fbbd12e7097e348d6a38622d364b1fa47ed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 10 Jan 2019 15:39:51 +0100 Subject: Add ability to unfederate a local video (on blacklist) --- server/controllers/api/videos/blacklist.ts | 17 +- server/controllers/api/videos/index.ts | 6 +- server/initializers/constants.ts | 2 +- .../migrations/0320-blacklist-unfederate.ts | 27 ++ server/lib/activitypub/share.ts | 16 +- .../validators/videos/video-blacklist.ts | 15 +- server/models/video/video-blacklist.ts | 21 +- server/tests/api/check-params/video-blacklist.ts | 118 ++++---- server/tests/api/videos/index.ts | 1 - .../tests/api/videos/video-blacklist-management.ts | 192 ------------- server/tests/api/videos/video-blacklist.ts | 299 ++++++++++++++++++--- 11 files changed, 407 insertions(+), 307 deletions(-) create mode 100644 server/initializers/migrations/0320-blacklist-unfederate.ts delete mode 100644 server/tests/api/videos/video-blacklist-management.ts (limited to 'server') diff --git a/server/controllers/api/videos/blacklist.ts b/server/controllers/api/videos/blacklist.ts index 9ef08812b..43b0516e7 100644 --- a/server/controllers/api/videos/blacklist.ts +++ b/server/controllers/api/videos/blacklist.ts @@ -18,6 +18,8 @@ import { VideoBlacklistModel } from '../../../models/video/video-blacklist' import { sequelizeTypescript } from '../../../initializers' import { Notifier } from '../../../lib/notifier' import { VideoModel } from '../../../models/video/video' +import { sendCreateVideo, sendDeleteVideo, sendUpdateVideo } from '../../../lib/activitypub/send' +import { federateVideoIfNeeded } from '../../../lib/activitypub' const blacklistRouter = express.Router() @@ -66,12 +68,17 @@ async function addVideoToBlacklist (req: express.Request, res: express.Response) const toCreate = { videoId: videoInstance.id, + unfederated: body.unfederate === true, reason: body.reason } const blacklist = await VideoBlacklistModel.create(toCreate) blacklist.Video = videoInstance + if (body.unfederate === true) { + await sendDeleteVideo(videoInstance, undefined) + } + Notifier.Instance.notifyOnVideoBlacklist(blacklist) logger.info('Video %s blacklisted.', res.locals.video.uuid) @@ -101,8 +108,14 @@ async function removeVideoFromBlacklistController (req: express.Request, res: ex const videoBlacklist = res.locals.videoBlacklist as VideoBlacklistModel const video: VideoModel = res.locals.video - await sequelizeTypescript.transaction(t => { - return videoBlacklist.destroy({ transaction: t }) + await sequelizeTypescript.transaction(async t => { + const unfederated = videoBlacklist.unfederated + await videoBlacklist.destroy({ transaction: t }) + + // Re federate the video + if (unfederated === true) { + await federateVideoIfNeeded(video, true, t) + } }) Notifier.Instance.notifyOnVideoUnblacklist(video) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 33521a8c1..28ac26598 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -364,7 +364,11 @@ async function updateVideo (req: express.Request, res: express.Response) { } const isNewVideo = wasPrivateVideo && videoInstanceUpdated.privacy !== VideoPrivacy.PRIVATE - await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) + + // Don't send update if the video was unfederated + if (!videoInstanceUpdated.VideoBlacklist || videoInstanceUpdated.VideoBlacklist.unfederated === false) { + await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) + } auditLogger.update( getAuditIdFromRes(res), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 4a88aef87..b18884eeb 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -16,7 +16,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 315 +const LAST_MIGRATION_VERSION = 320 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0320-blacklist-unfederate.ts b/server/initializers/migrations/0320-blacklist-unfederate.ts new file mode 100644 index 000000000..6fb7bbb90 --- /dev/null +++ b/server/initializers/migrations/0320-blacklist-unfederate.ts @@ -0,0 +1,27 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize +}): Promise { + + { + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false + } + + await utils.queryInterface.addColumn('videoBlacklist', 'unfederated', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 5dcba778c..170e49238 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -78,7 +78,7 @@ async function shareByServer (video: VideoModel, t: Transaction) { const serverActor = await getServerActor() const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video) - return VideoShareModel.findOrCreate({ + const [ serverShare ] = await VideoShareModel.findOrCreate({ defaults: { actorId: serverActor.id, videoId: video.id, @@ -88,16 +88,14 @@ async function shareByServer (video: VideoModel, t: Transaction) { url: serverShareUrl }, transaction: t - }).then(([ serverShare, created ]) => { - if (created) return sendVideoAnnounce(serverActor, serverShare, video, t) - - return undefined }) + + return sendVideoAnnounce(serverActor, serverShare, video, t) } async function shareByVideoChannel (video: VideoModel, t: Transaction) { const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) - return VideoShareModel.findOrCreate({ + const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ defaults: { actorId: video.VideoChannel.actorId, videoId: video.id, @@ -107,11 +105,9 @@ async function shareByVideoChannel (video: VideoModel, t: Transaction) { url: videoChannelShareUrl }, transaction: t - }).then(([ videoChannelShare, created ]) => { - if (created) return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) - - return undefined }) + + return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) } async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) { diff --git a/server/middlewares/validators/videos/video-blacklist.ts b/server/middlewares/validators/videos/video-blacklist.ts index 13da7acff..2688f63ae 100644 --- a/server/middlewares/validators/videos/video-blacklist.ts +++ b/server/middlewares/validators/videos/video-blacklist.ts @@ -1,10 +1,11 @@ import * as express from 'express' import { body, param } from 'express-validator/check' -import { isIdOrUUIDValid } from '../../../helpers/custom-validators/misc' +import { isBooleanValid, isIdOrUUIDValid } from '../../../helpers/custom-validators/misc' import { isVideoExist } from '../../../helpers/custom-validators/videos' import { logger } from '../../../helpers/logger' import { areValidationErrors } from '../utils' import { isVideoBlacklistExist, isVideoBlacklistReasonValid } from '../../../helpers/custom-validators/video-blacklist' +import { VideoModel } from '../../../models/video/video' const videosBlacklistRemoveValidator = [ param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid videoId'), @@ -22,6 +23,10 @@ const videosBlacklistRemoveValidator = [ const videosBlacklistAddValidator = [ param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid videoId'), + body('unfederate') + .optional() + .toBoolean() + .custom(isBooleanValid).withMessage('Should have a valid unfederate boolean'), body('reason') .optional() .custom(isVideoBlacklistReasonValid).withMessage('Should have a valid reason'), @@ -32,6 +37,14 @@ const videosBlacklistAddValidator = [ if (areValidationErrors(req, res)) return if (!await isVideoExist(req.params.videoId, res)) return + const video: VideoModel = res.locals.video + if (req.body.unfederate === true && video.remote === true) { + return res + .status(409) + .send({ error: 'You cannot unfederate a remote video.' }) + .end() + } + return next() } ] diff --git a/server/models/video/video-blacklist.ts b/server/models/video/video-blacklist.ts index 23e992685..3b567e488 100644 --- a/server/models/video/video-blacklist.ts +++ b/server/models/video/video-blacklist.ts @@ -1,21 +1,7 @@ -import { - AfterCreate, - AfterDestroy, - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { getSortOnModel, SortType, throwIfNotValid } from '../utils' import { VideoModel } from './video' import { isVideoBlacklistReasonValid } from '../../helpers/custom-validators/video-blacklist' -import { Emailer } from '../../lib/emailer' import { VideoBlacklist } from '../../../shared/models/videos' import { CONSTRAINTS_FIELDS } from '../../initializers' @@ -35,6 +21,10 @@ export class VideoBlacklistModel extends Model { @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_BLACKLIST.REASON.max)) reason: string + @AllowNull(false) + @Column + unfederated: boolean + @CreatedAt createdAt: Date @@ -93,6 +83,7 @@ export class VideoBlacklistModel extends Model { createdAt: this.createdAt, updatedAt: this.updatedAt, reason: this.reason, + unfederated: this.unfederated, video: { id: video.id, diff --git a/server/tests/api/check-params/video-blacklist.ts b/server/tests/api/check-params/video-blacklist.ts index 8e1206db3..6b82643f4 100644 --- a/server/tests/api/check-params/video-blacklist.ts +++ b/server/tests/api/check-params/video-blacklist.ts @@ -4,17 +4,20 @@ import 'mocha' import { createUser, + doubleFollow, + flushAndRunMultipleServers, flushTests, - getBlacklistedVideosList, getVideo, getVideoWithToken, + getBlacklistedVideosList, + getVideo, + getVideoWithToken, killallServers, makePostBodyRequest, makePutBodyRequest, removeVideoFromBlacklist, - runServer, ServerInfo, setAccessTokensToServers, uploadVideo, - userLogin + userLogin, waitJobs } from '../../../../shared/utils' import { checkBadCountPagination, @@ -25,8 +28,9 @@ import { VideoDetails } from '../../../../shared/models/videos' import { expect } from 'chai' describe('Test video blacklist API validators', function () { - let server: ServerInfo + let servers: ServerInfo[] let notBlacklistedVideoId: number + let remoteVideoUUID: string let userAccessToken1 = '' let userAccessToken2 = '' @@ -36,75 +40,89 @@ describe('Test video blacklist API validators', function () { this.timeout(120000) await flushTests() + servers = await flushAndRunMultipleServers(2) - server = await runServer(1) - - await setAccessTokensToServers([ server ]) + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) { const username = 'user1' const password = 'my super password' - await createUser(server.url, server.accessToken, username, password) - userAccessToken1 = await userLogin(server, { username, password }) + await createUser(servers[0].url, servers[0].accessToken, username, password) + userAccessToken1 = await userLogin(servers[0], { username, password }) } { const username = 'user2' const password = 'my super password' - await createUser(server.url, server.accessToken, username, password) - userAccessToken2 = await userLogin(server, { username, password }) + await createUser(servers[0].url, servers[0].accessToken, username, password) + userAccessToken2 = await userLogin(servers[0], { username, password }) } { - const res = await uploadVideo(server.url, userAccessToken1, {}) - server.video = res.body.video + const res = await uploadVideo(servers[0].url, userAccessToken1, {}) + servers[0].video = res.body.video } { - const res = await uploadVideo(server.url, server.accessToken, {}) + const res = await uploadVideo(servers[0].url, servers[0].accessToken, {}) notBlacklistedVideoId = res.body.video.uuid } + + { + const res = await uploadVideo(servers[1].url, servers[1].accessToken, {}) + remoteVideoUUID = res.body.video.uuid + } + + await waitJobs(servers) }) describe('When adding a video in blacklist', function () { const basePath = '/api/v1/videos/' it('Should fail with nothing', async function () { - const path = basePath + server.video + '/blacklist' + const path = basePath + servers[0].video + '/blacklist' const fields = {} - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) }) it('Should fail with a wrong video', async function () { const wrongPath = '/api/v1/videos/blabla/blacklist' const fields = {} - await makePostBodyRequest({ url: server.url, path: wrongPath, token: server.accessToken, fields }) + await makePostBodyRequest({ url: servers[0].url, path: wrongPath, token: servers[0].accessToken, fields }) }) it('Should fail with a non authenticated user', async function () { - const path = basePath + server.video + '/blacklist' + const path = basePath + servers[0].video + '/blacklist' const fields = {} - await makePostBodyRequest({ url: server.url, path, token: 'hello', fields, statusCodeExpected: 401 }) + await makePostBodyRequest({ url: servers[0].url, path, token: 'hello', fields, statusCodeExpected: 401 }) }) it('Should fail with a non admin user', async function () { - const path = basePath + server.video + '/blacklist' + const path = basePath + servers[0].video + '/blacklist' const fields = {} - await makePostBodyRequest({ url: server.url, path, token: userAccessToken2, fields, statusCodeExpected: 403 }) + await makePostBodyRequest({ url: servers[0].url, path, token: userAccessToken2, fields, statusCodeExpected: 403 }) }) it('Should fail with an invalid reason', async function () { - const path = basePath + server.video.uuid + '/blacklist' + const path = basePath + servers[0].video.uuid + '/blacklist' const fields = { reason: 'a'.repeat(305) } - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) + }) + + it('Should fail to unfederate a remote video', async function () { + const path = basePath + remoteVideoUUID + '/blacklist' + const fields = { unfederate: true } + + await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields, statusCodeExpected: 409 }) }) it('Should succeed with the correct params', async function () { - const path = basePath + server.video.uuid + '/blacklist' + const path = basePath + servers[0].video.uuid + '/blacklist' const fields = { } - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields, statusCodeExpected: 204 }) + await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields, statusCodeExpected: 204 }) }) }) @@ -114,61 +132,61 @@ describe('Test video blacklist API validators', function () { it('Should fail with a wrong video', async function () { const wrongPath = '/api/v1/videos/blabla/blacklist' const fields = {} - await makePutBodyRequest({ url: server.url, path: wrongPath, token: server.accessToken, fields }) + await makePutBodyRequest({ url: servers[0].url, path: wrongPath, token: servers[0].accessToken, fields }) }) it('Should fail with a video not blacklisted', async function () { const path = '/api/v1/videos/' + notBlacklistedVideoId + '/blacklist' const fields = {} - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields, statusCodeExpected: 404 }) + await makePutBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields, statusCodeExpected: 404 }) }) it('Should fail with a non authenticated user', async function () { - const path = basePath + server.video + '/blacklist' + const path = basePath + servers[0].video + '/blacklist' const fields = {} - await makePutBodyRequest({ url: server.url, path, token: 'hello', fields, statusCodeExpected: 401 }) + await makePutBodyRequest({ url: servers[0].url, path, token: 'hello', fields, statusCodeExpected: 401 }) }) it('Should fail with a non admin user', async function () { - const path = basePath + server.video + '/blacklist' + const path = basePath + servers[0].video + '/blacklist' const fields = {} - await makePutBodyRequest({ url: server.url, path, token: userAccessToken2, fields, statusCodeExpected: 403 }) + await makePutBodyRequest({ url: servers[0].url, path, token: userAccessToken2, fields, statusCodeExpected: 403 }) }) it('Should fail with an invalid reason', async function () { - const path = basePath + server.video.uuid + '/blacklist' + const path = basePath + servers[0].video.uuid + '/blacklist' const fields = { reason: 'a'.repeat(305) } - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + await makePutBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) }) it('Should succeed with the correct params', async function () { - const path = basePath + server.video.uuid + '/blacklist' + const path = basePath + servers[0].video.uuid + '/blacklist' const fields = { reason: 'hello' } - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields, statusCodeExpected: 204 }) + await makePutBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields, statusCodeExpected: 204 }) }) }) describe('When getting blacklisted video', function () { it('Should fail with a non authenticated user', async function () { - await getVideo(server.url, server.video.uuid, 401) + await getVideo(servers[0].url, servers[0].video.uuid, 401) }) it('Should fail with another user', async function () { - await getVideoWithToken(server.url, userAccessToken2, server.video.uuid, 403) + await getVideoWithToken(servers[0].url, userAccessToken2, servers[0].video.uuid, 403) }) it('Should succeed with the owner authenticated user', async function () { - const res = await getVideoWithToken(server.url, userAccessToken1, server.video.uuid, 200) + const res = await getVideoWithToken(servers[0].url, userAccessToken1, servers[0].video.uuid, 200) const video: VideoDetails = res.body expect(video.blacklisted).to.be.true }) it('Should succeed with an admin', async function () { - const res = await getVideoWithToken(server.url, server.accessToken, server.video.uuid, 200) + const res = await getVideoWithToken(servers[0].url, servers[0].accessToken, servers[0].video.uuid, 200) const video: VideoDetails = res.body expect(video.blacklisted).to.be.true @@ -177,24 +195,24 @@ describe('Test video blacklist API validators', function () { describe('When removing a video in blacklist', function () { it('Should fail with a non authenticated user', async function () { - await removeVideoFromBlacklist(server.url, 'fake token', server.video.uuid, 401) + await removeVideoFromBlacklist(servers[0].url, 'fake token', servers[0].video.uuid, 401) }) it('Should fail with a non admin user', async function () { - await removeVideoFromBlacklist(server.url, userAccessToken2, server.video.uuid, 403) + await removeVideoFromBlacklist(servers[0].url, userAccessToken2, servers[0].video.uuid, 403) }) it('Should fail with an incorrect id', async function () { - await removeVideoFromBlacklist(server.url, server.accessToken, 'hello', 400) + await removeVideoFromBlacklist(servers[0].url, servers[0].accessToken, 'hello', 400) }) it('Should fail with a not blacklisted video', async function () { // The video was not added to the blacklist so it should fail - await removeVideoFromBlacklist(server.url, server.accessToken, notBlacklistedVideoId, 404) + await removeVideoFromBlacklist(servers[0].url, servers[0].accessToken, notBlacklistedVideoId, 404) }) it('Should succeed with the correct params', async function () { - await removeVideoFromBlacklist(server.url, server.accessToken, server.video.uuid, 204) + await removeVideoFromBlacklist(servers[0].url, servers[0].accessToken, servers[0].video.uuid, 204) }) }) @@ -202,28 +220,28 @@ describe('Test video blacklist API validators', function () { const basePath = '/api/v1/videos/blacklist/' it('Should fail with a non authenticated user', async function () { - await getBlacklistedVideosList(server.url, 'fake token', 401) + await getBlacklistedVideosList(servers[0].url, 'fake token', 401) }) it('Should fail with a non admin user', async function () { - await getBlacklistedVideosList(server.url, userAccessToken2, 403) + await getBlacklistedVideosList(servers[0].url, userAccessToken2, 403) }) it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, basePath, server.accessToken) + await checkBadStartPagination(servers[0].url, basePath, servers[0].accessToken) }) it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, basePath, server.accessToken) + await checkBadCountPagination(servers[0].url, basePath, servers[0].accessToken) }) it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, basePath, server.accessToken) + await checkBadSortPagination(servers[0].url, basePath, servers[0].accessToken) }) }) after(async function () { - killallServers([ server ]) + killallServers(servers) // Keep the logs if the test failed if (this['ok']) { diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 9bdb78491..97f467aae 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -3,7 +3,6 @@ import './services' import './single-server' import './video-abuse' import './video-blacklist' -import './video-blacklist-management' import './video-captions' import './video-change-ownership' import './video-channels' diff --git a/server/tests/api/videos/video-blacklist-management.ts b/server/tests/api/videos/video-blacklist-management.ts deleted file mode 100644 index 61411e30d..000000000 --- a/server/tests/api/videos/video-blacklist-management.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* tslint:disable:no-unused-expression */ - -import * as chai from 'chai' -import { orderBy } from 'lodash' -import 'mocha' -import { - addVideoToBlacklist, - flushAndRunMultipleServers, - getBlacklistedVideosList, - getMyVideos, - getSortedBlacklistedVideosList, - getVideosList, - killallServers, - removeVideoFromBlacklist, - ServerInfo, - setAccessTokensToServers, - updateVideoBlacklist, - uploadVideo -} from '../../../../shared/utils/index' -import { doubleFollow } from '../../../../shared/utils/server/follows' -import { waitJobs } from '../../../../shared/utils/server/jobs' -import { VideoAbuse } from '../../../../shared/models/videos' - -const expect = chai.expect - -describe('Test video blacklist management', function () { - let servers: ServerInfo[] = [] - let videoId: number - - async function blacklistVideosOnServer (server: ServerInfo) { - const res = await getVideosList(server.url) - - const videos = res.body.data - for (let video of videos) { - await addVideoToBlacklist(server.url, server.accessToken, video.id, 'super reason') - } - } - - before(async function () { - this.timeout(50000) - - // Run servers - servers = await flushAndRunMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - - // Upload 2 videos on server 2 - await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'My 1st video', description: 'A video on server 2' }) - await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'My 2nd video', description: 'A video on server 2' }) - - // Wait videos propagation, server 2 has transcoding enabled - await waitJobs(servers) - - // Blacklist the two videos on server 1 - await blacklistVideosOnServer(servers[0]) - }) - - describe('When listing blacklisted videos', function () { - it('Should display all the blacklisted videos', async function () { - const res = await getBlacklistedVideosList(servers[0].url, servers[0].accessToken) - - expect(res.body.total).to.equal(2) - - const blacklistedVideos = res.body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - for (const blacklistedVideo of blacklistedVideos) { - expect(blacklistedVideo.reason).to.equal('super reason') - videoId = blacklistedVideo.video.id - } - }) - - it('Should get the correct sort when sorting by descending id', async function () { - const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-id') - expect(res.body.total).to.equal(2) - - const blacklistedVideos = res.body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - const result = orderBy(res.body.data, [ 'id' ], [ 'desc' ]) - - expect(blacklistedVideos).to.deep.equal(result) - }) - - it('Should get the correct sort when sorting by descending video name', async function () { - const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') - expect(res.body.total).to.equal(2) - - const blacklistedVideos = res.body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - const result = orderBy(res.body.data, [ 'name' ], [ 'desc' ]) - - expect(blacklistedVideos).to.deep.equal(result) - }) - - it('Should get the correct sort when sorting by ascending creation date', async function () { - const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, 'createdAt') - expect(res.body.total).to.equal(2) - - const blacklistedVideos = res.body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - const result = orderBy(res.body.data, [ 'createdAt' ]) - - expect(blacklistedVideos).to.deep.equal(result) - }) - }) - - describe('When updating blacklisted videos', function () { - it('Should change the reason', async function () { - await updateVideoBlacklist(servers[0].url, servers[0].accessToken, videoId, 'my super reason updated') - - const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') - const video = res.body.data.find(b => b.video.id === videoId) - - expect(video.reason).to.equal('my super reason updated') - }) - }) - - describe('When listing my videos', function () { - it('Should display blacklisted videos', async function () { - await blacklistVideosOnServer(servers[1]) - - const res = await getMyVideos(servers[1].url, servers[1].accessToken, 0, 5) - - expect(res.body.total).to.equal(2) - expect(res.body.data).to.have.lengthOf(2) - - for (const video of res.body.data) { - expect(video.blacklisted).to.be.true - expect(video.blacklistedReason).to.equal('super reason') - } - }) - }) - - describe('When removing a blacklisted video', function () { - let videoToRemove: VideoAbuse - let blacklist = [] - - it('Should not have any video in videos list on server 1', async function () { - const res = await getVideosList(servers[0].url) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(0) - }) - - it('Should remove a video from the blacklist on server 1', async function () { - // Get one video in the blacklist - const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') - videoToRemove = res.body.data[0] - blacklist = res.body.data.slice(1) - - // Remove it - await removeVideoFromBlacklist(servers[0].url, servers[0].accessToken, videoToRemove.video.id) - }) - - it('Should have the ex-blacklisted video in videos list on server 1', async function () { - const res = await getVideosList(servers[0].url) - expect(res.body.total).to.equal(1) - - const videos = res.body.data - expect(videos).to.be.an('array') - expect(videos.length).to.equal(1) - - expect(videos[0].name).to.equal(videoToRemove.video.name) - expect(videos[0].id).to.equal(videoToRemove.video.id) - }) - - it('Should not have the ex-blacklisted video in videos blacklist list on server 1', async function () { - const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') - expect(res.body.total).to.equal(1) - - const videos = res.body.data - expect(videos).to.be.an('array') - expect(videos.length).to.equal(1) - expect(videos).to.deep.equal(blacklist) - }) - }) - - after(async function () { - killallServers(servers) - }) -}) diff --git a/server/tests/api/videos/video-blacklist.ts b/server/tests/api/videos/video-blacklist.ts index 1cce82d2a..d39ad63b4 100644 --- a/server/tests/api/videos/video-blacklist.ts +++ b/server/tests/api/videos/video-blacklist.ts @@ -1,24 +1,43 @@ /* tslint:disable:no-unused-expression */ import * as chai from 'chai' +import { orderBy } from 'lodash' import 'mocha' import { addVideoToBlacklist, flushAndRunMultipleServers, + getBlacklistedVideosList, + getMyVideos, + getSortedBlacklistedVideosList, getVideosList, killallServers, + removeVideoFromBlacklist, searchVideo, ServerInfo, setAccessTokensToServers, - uploadVideo + updateVideo, + updateVideoBlacklist, + uploadVideo, + viewVideo } from '../../../../shared/utils/index' import { doubleFollow } from '../../../../shared/utils/server/follows' import { waitJobs } from '../../../../shared/utils/server/jobs' +import { VideoBlacklist } from '../../../../shared/models/videos' const expect = chai.expect -describe('Test video blacklists', function () { +describe('Test video blacklist management', function () { let servers: ServerInfo[] = [] + let videoId: number + + async function blacklistVideosOnServer (server: ServerInfo) { + const res = await getVideosList(server.url) + + const videos = res.body.data + for (let video of videos) { + await addVideoToBlacklist(server.url, server.accessToken, video.id, 'super reason') + } + } before(async function () { this.timeout(50000) @@ -32,58 +51,270 @@ describe('Test video blacklists', function () { // Server 1 and server 2 follow each other await doubleFollow(servers[0], servers[1]) - // Upload a video on server 2 - const videoAttributes = { - name: 'my super name for server 2', - description: 'my super description for server 2' - } - await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) + // Upload 2 videos on server 2 + await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'My 1st video', description: 'A video on server 2' }) + await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'My 2nd video', description: 'A video on server 2' }) // Wait videos propagation, server 2 has transcoding enabled await waitJobs(servers) - const res = await getVideosList(servers[0].url) - const videos = res.body.data + // Blacklist the two videos on server 1 + await blacklistVideosOnServer(servers[0]) + }) + + describe('When listing/searching videos', function () { - expect(videos.length).to.equal(1) + it('Should not have the video blacklisted in videos list/search on server 1', async function () { + { + const res = await getVideosList(servers[ 0 ].url) - servers[0].remoteVideo = videos.find(video => video.name === 'my super name for server 2') + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + } + + { + const res = await searchVideo(servers[ 0 ].url, 'name') + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + } + }) + + it('Should have the blacklisted video in videos list/search on server 2', async function () { + { + const res = await getVideosList(servers[ 1 ].url) + + expect(res.body.total).to.equal(2) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(2) + } + + { + const res = await searchVideo(servers[ 1 ].url, 'video') + + expect(res.body.total).to.equal(2) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(2) + } + }) }) - it('Should blacklist a remote video on server 1', async function () { - await addVideoToBlacklist(servers[0].url, servers[0].accessToken, servers[0].remoteVideo.id) + describe('When listing blacklisted videos', function () { + it('Should display all the blacklisted videos', async function () { + const res = await getBlacklistedVideosList(servers[0].url, servers[0].accessToken) + + expect(res.body.total).to.equal(2) + + const blacklistedVideos = res.body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + for (const blacklistedVideo of blacklistedVideos) { + expect(blacklistedVideo.reason).to.equal('super reason') + videoId = blacklistedVideo.video.id + } + }) + + it('Should get the correct sort when sorting by descending id', async function () { + const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-id') + expect(res.body.total).to.equal(2) + + const blacklistedVideos = res.body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + const result = orderBy(res.body.data, [ 'id' ], [ 'desc' ]) + + expect(blacklistedVideos).to.deep.equal(result) + }) + + it('Should get the correct sort when sorting by descending video name', async function () { + const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') + expect(res.body.total).to.equal(2) + + const blacklistedVideos = res.body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + const result = orderBy(res.body.data, [ 'name' ], [ 'desc' ]) + + expect(blacklistedVideos).to.deep.equal(result) + }) + + it('Should get the correct sort when sorting by ascending creation date', async function () { + const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, 'createdAt') + expect(res.body.total).to.equal(2) + + const blacklistedVideos = res.body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + const result = orderBy(res.body.data, [ 'createdAt' ]) + + expect(blacklistedVideos).to.deep.equal(result) + }) }) - it('Should not have the video blacklisted in videos list on server 1', async function () { - const res = await getVideosList(servers[0].url) + describe('When updating blacklisted videos', function () { + it('Should change the reason', async function () { + await updateVideoBlacklist(servers[0].url, servers[0].accessToken, videoId, 'my super reason updated') + + const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') + const video = res.body.data.find(b => b.video.id === videoId) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(0) + expect(video.reason).to.equal('my super reason updated') + }) }) - it('Should not have the video blacklisted in videos search on server 1', async function () { - const res = await searchVideo(servers[0].url, 'name') + describe('When listing my videos', function () { + it('Should display blacklisted videos', async function () { + await blacklistVideosOnServer(servers[1]) + + const res = await getMyVideos(servers[1].url, servers[1].accessToken, 0, 5) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(0) + expect(res.body.total).to.equal(2) + expect(res.body.data).to.have.lengthOf(2) + + for (const video of res.body.data) { + expect(video.blacklisted).to.be.true + expect(video.blacklistedReason).to.equal('super reason') + } + }) }) - it('Should have the blacklisted video in videos list on server 2', async function () { - const res = await getVideosList(servers[1].url) + describe('When removing a blacklisted video', function () { + let videoToRemove: VideoBlacklist + let blacklist = [] + + it('Should not have any video in videos list on server 1', async function () { + const res = await getVideosList(servers[0].url) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + }) + + it('Should remove a video from the blacklist on server 1', async function () { + // Get one video in the blacklist + const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') + videoToRemove = res.body.data[0] + blacklist = res.body.data.slice(1) + + // Remove it + await removeVideoFromBlacklist(servers[0].url, servers[0].accessToken, videoToRemove.video.id) + }) + + it('Should have the ex-blacklisted video in videos list on server 1', async function () { + const res = await getVideosList(servers[0].url) + expect(res.body.total).to.equal(1) + + const videos = res.body.data + expect(videos).to.be.an('array') + expect(videos.length).to.equal(1) + + expect(videos[0].name).to.equal(videoToRemove.video.name) + expect(videos[0].id).to.equal(videoToRemove.video.id) + }) + + it('Should not have the ex-blacklisted video in videos blacklist list on server 1', async function () { + const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, '-name') + expect(res.body.total).to.equal(1) - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) + const videos = res.body.data + expect(videos).to.be.an('array') + expect(videos.length).to.equal(1) + expect(videos).to.deep.equal(blacklist) + }) }) - it('Should have the video blacklisted in videos search on server 2', async function () { - const res = await searchVideo(servers[1].url, 'name') + describe('When blacklisting local videos', function () { + let video3UUID: string + let video4UUID: string + + before(async function () { + this.timeout(10000) + + { + const res = await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'Video 3' }) + video3UUID = res.body.video.uuid + } + { + const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'Video 4' }) + video4UUID = res.body.video.uuid + } + + await waitJobs(servers) + }) + + it('Should blacklist video 3 and keep it federated', async function () { + this.timeout(10000) + + await addVideoToBlacklist(servers[ 0 ].url, servers[ 0 ].accessToken, video3UUID, 'super reason', false) + + await waitJobs(servers) + + { + const res = await getVideosList(servers[ 0 ].url) + expect(res.body.data.find(v => v.uuid === video3UUID)).to.be.undefined + } + + { + const res = await getVideosList(servers[ 1 ].url) + expect(res.body.data.find(v => v.uuid === video3UUID)).to.not.be.undefined + } + }) + + it('Should unfederate the video', async function () { + this.timeout(10000) + + await addVideoToBlacklist(servers[ 0 ].url, servers[ 0 ].accessToken, video4UUID, 'super reason', true) + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideosList(server.url) + expect(res.body.data.find(v => v.uuid === video4UUID)).to.be.undefined + } + }) + + it('Should have the video unfederated even after an Update AP message', async function () { + this.timeout(10000) + + await updateVideo(servers[ 0 ].url, servers[ 0 ].accessToken, video4UUID, { description: 'super description' }) + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideosList(server.url) + expect(res.body.data.find(v => v.uuid === video4UUID)).to.be.undefined + } + }) + + it('Should have the correct video blacklist unfederate attribute', async function () { + const res = await getSortedBlacklistedVideosList(servers[0].url, servers[0].accessToken, 'createdAt') + + const blacklistedVideos: VideoBlacklist[] = res.body.data + const video3Blacklisted = blacklistedVideos.find(b => b.video.uuid === video3UUID) + const video4Blacklisted = blacklistedVideos.find(b => b.video.uuid === video4UUID) + + expect(video3Blacklisted.unfederated).to.be.false + expect(video4Blacklisted.unfederated).to.be.true + }) + + it('Should remove the video from blacklist and refederate the video', async function () { + this.timeout(10000) + + await removeVideoFromBlacklist(servers[ 0 ].url, servers[ 0 ].accessToken, video4UUID) + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideosList(server.url) + expect(res.body.data.find(v => v.uuid === video4UUID)).to.not.be.undefined + } + }) - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) }) after(async function () { -- cgit v1.2.3 From c04eb647db4f543a31a8100c1ec9a86c700bca6a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 10 Jan 2019 16:00:23 +0100 Subject: Use origin video url in canonical tag --- server/lib/client-html.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'server') diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 1875ec1fc..b2c376e20 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -1,7 +1,7 @@ import * as express from 'express' import * as Bluebird from 'bluebird' import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' -import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, STATIC_PATHS } from '../initializers' +import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE } from '../initializers' import { join } from 'path' import { escapeHTML } from '../helpers/core-utils' import { VideoModel } from '../models/video/video' @@ -187,8 +187,8 @@ export class ClientHtml { // Schema.org tagsString += `` - // SEO - tagsString += `` + // SEO, use origin video url so Google does not index remote videos + tagsString += `` return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) } -- cgit v1.2.3 From 9b4b15f91c485f9a7fe2ed314b4101f4b7506b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20B=C3=A9ranger?= <43744761+auberanger@users.noreply.github.com> Date: Mon, 14 Jan 2019 09:06:48 +0100 Subject: WIP : Indicate to users how "trending" works (#1458) * Get the INTERVAL_DAYS const in the video-trending component * Change Trending section title * Add a tooltip to explain how trending section works * Minor CSS fix for the my-feed popover next to the titlepage --- server/controllers/api/config.ts | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'server') diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index dd06a0597..255026f46 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -120,6 +120,11 @@ async function getConfig (req: express.Request, res: express.Response) { user: { videoQuota: CONFIG.USER.VIDEO_QUOTA, videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY + }, + trending: { + videos: { + intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS + } } } -- cgit v1.2.3 From b4593cd7ff34b94b60f6bfa0b57e371d74d63aa2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 10:24:49 +0100 Subject: Warn user when they want to delete a channel Because they will not be able to create another channel with the same actor name --- server/lib/activitypub/actor.ts | 2 +- server/lib/activitypub/process/process.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'server') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f7bf7c65a..f80296725 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -296,7 +296,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe const actorJSON: ActivityPubActor = requestResult.body if (isActorObjectValid(actorJSON) === false) { - logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON }) + logger.debug('Remote actor JSON is not valid.', { actorJSON }) return { result: undefined, statusCode: requestResult.response.statusCode } } diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index bcc5cac7a..2479d5da2 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -35,7 +35,7 @@ async function processActivities ( const actorsCache: { [ url: string ]: ActorModel } = {} for (const activity of activities) { - if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { + if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) continue } -- cgit v1.2.3 From cf405589f06a9ac9d5a05b09bf2183fbf88d56d7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 10:44:59 +0100 Subject: Move subscriptions controllers in its own file --- server/controllers/api/users/index.ts | 2 + server/controllers/api/users/me.ts | 156 +-------------------- server/controllers/api/users/my-subscriptions.ts | 170 +++++++++++++++++++++++ 3 files changed, 174 insertions(+), 154 deletions(-) create mode 100644 server/controllers/api/users/my-subscriptions.ts (limited to 'server') diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index 9e6a019f6..dbe0718d4 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -41,6 +41,7 @@ import { myBlocklistRouter } from './my-blocklist' import { myVideosHistoryRouter } from './my-history' import { myNotificationsRouter } from './my-notifications' import { Notifier } from '../../../lib/notifier' +import { mySubscriptionsRouter } from './my-subscriptions' const auditLogger = auditLoggerFactory('users') @@ -58,6 +59,7 @@ const askSendEmailLimiter = new RateLimit({ const usersRouter = express.Router() usersRouter.use('/', myNotificationsRouter) +usersRouter.use('/', mySubscriptionsRouter) usersRouter.use('/', myBlocklistRouter) usersRouter.use('/', myVideosHistoryRouter) usersRouter.use('/', meRouter) diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 8a3208160..94a2b8732 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -8,36 +8,23 @@ import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, - commonVideosFiltersValidator, paginationValidator, setDefaultPagination, setDefaultSort, - userSubscriptionAddValidator, - userSubscriptionGetValidator, usersUpdateMeValidator, usersVideoRatingValidator } from '../../../middlewares' -import { - areSubscriptionsExistValidator, - deleteMeValidator, - userSubscriptionsSortValidator, - videoImportsSortValidator, - videosSortValidator -} from '../../../middlewares/validators' +import { deleteMeValidator, videoImportsSortValidator, videosSortValidator } from '../../../middlewares/validators' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { UserModel } from '../../../models/account/user' import { VideoModel } from '../../../models/video/video' import { VideoSortField } from '../../../../client/src/app/shared/video/sort-field.type' -import { buildNSFWFilter, createReqFiles } from '../../../helpers/express-utils' +import { createReqFiles } from '../../../helpers/express-utils' import { UserVideoQuota } from '../../../../shared/models/users/user-video-quota.model' import { updateAvatarValidator } from '../../../middlewares/validators/avatar' import { updateActorAvatarFile } from '../../../lib/avatar' import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger' import { VideoImportModel } from '../../../models/video/video-import' -import { VideoFilter } from '../../../../shared/models/videos/video-query.type' -import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { JobQueue } from '../../../lib/job-queue' -import { logger } from '../../../helpers/logger' import { AccountModel } from '../../../models/account/account' const auditLogger = auditLoggerFactory('users-me') @@ -98,51 +85,6 @@ meRouter.post('/me/avatar/pick', asyncRetryTransactionMiddleware(updateMyAvatar) ) -// ##### Subscriptions part ##### - -meRouter.get('/me/subscriptions/videos', - authenticate, - paginationValidator, - videosSortValidator, - setDefaultSort, - setDefaultPagination, - commonVideosFiltersValidator, - asyncMiddleware(getUserSubscriptionVideos) -) - -meRouter.get('/me/subscriptions/exist', - authenticate, - areSubscriptionsExistValidator, - asyncMiddleware(areSubscriptionsExist) -) - -meRouter.get('/me/subscriptions', - authenticate, - paginationValidator, - userSubscriptionsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(getUserSubscriptions) -) - -meRouter.post('/me/subscriptions', - authenticate, - userSubscriptionAddValidator, - asyncMiddleware(addUserSubscription) -) - -meRouter.get('/me/subscriptions/:uri', - authenticate, - userSubscriptionGetValidator, - getUserSubscription -) - -meRouter.delete('/me/subscriptions/:uri', - authenticate, - userSubscriptionGetValidator, - asyncRetryTransactionMiddleware(deleteUserSubscription) -) - // --------------------------------------------------------------------------- export { @@ -151,100 +93,6 @@ export { // --------------------------------------------------------------------------- -async function areSubscriptionsExist (req: express.Request, res: express.Response) { - const uris = req.query.uris as string[] - const user = res.locals.oauth.token.User as UserModel - - const handles = uris.map(u => { - let [ name, host ] = u.split('@') - if (host === CONFIG.WEBSERVER.HOST) host = null - - return { name, host, uri: u } - }) - - const results = await ActorFollowModel.listSubscribedIn(user.Account.Actor.id, handles) - - const existObject: { [id: string ]: boolean } = {} - for (const handle of handles) { - const obj = results.find(r => { - const server = r.ActorFollowing.Server - - return r.ActorFollowing.preferredUsername === handle.name && - ( - (!server && !handle.host) || - (server.host === handle.host) - ) - }) - - existObject[handle.uri] = obj !== undefined - } - - return res.json(existObject) -} - -async function addUserSubscription (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User as UserModel - const [ name, host ] = req.body.uri.split('@') - - const payload = { - name, - host, - followerActorId: user.Account.Actor.id - } - - JobQueue.Instance.createJob({ type: 'activitypub-follow', payload }) - .catch(err => logger.error('Cannot create follow job for subscription %s.', req.body.uri, err)) - - return res.status(204).end() -} - -function getUserSubscription (req: express.Request, res: express.Response) { - const subscription: ActorFollowModel = res.locals.subscription - - return res.json(subscription.ActorFollowing.VideoChannel.toFormattedJSON()) -} - -async function deleteUserSubscription (req: express.Request, res: express.Response) { - const subscription: ActorFollowModel = res.locals.subscription - - await sequelizeTypescript.transaction(async t => { - return subscription.destroy({ transaction: t }) - }) - - return res.type('json').status(204).end() -} - -async function getUserSubscriptions (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User as UserModel - const actorId = user.Account.Actor.id - - const resultList = await ActorFollowModel.listSubscriptionsForApi(actorId, req.query.start, req.query.count, req.query.sort) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function getUserSubscriptionVideos (req: express.Request, res: express.Response, next: express.NextFunction) { - const user = res.locals.oauth.token.User as UserModel - const resultList = await VideoModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - includeLocalVideos: false, - categoryOneOf: req.query.categoryOneOf, - licenceOneOf: req.query.licenceOneOf, - languageOneOf: req.query.languageOneOf, - tagsOneOf: req.query.tagsOneOf, - tagsAllOf: req.query.tagsAllOf, - nsfw: buildNSFWFilter(res, req.query.nsfw), - filter: req.query.filter as VideoFilter, - withFiles: false, - followerActorId: user.Account.Actor.id, - user - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - async function getUserVideos (req: express.Request, res: express.Response, next: express.NextFunction) { const user = res.locals.oauth.token.User as UserModel const resultList = await VideoModel.listUserVideosForApi( diff --git a/server/controllers/api/users/my-subscriptions.ts b/server/controllers/api/users/my-subscriptions.ts new file mode 100644 index 000000000..accca6d52 --- /dev/null +++ b/server/controllers/api/users/my-subscriptions.ts @@ -0,0 +1,170 @@ +import * as express from 'express' +import 'multer' +import { getFormattedObjects } from '../../../helpers/utils' +import { CONFIG, sequelizeTypescript } from '../../../initializers' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + commonVideosFiltersValidator, + paginationValidator, + setDefaultPagination, + setDefaultSort, + userSubscriptionAddValidator, + userSubscriptionGetValidator +} from '../../../middlewares' +import { areSubscriptionsExistValidator, userSubscriptionsSortValidator, videosSortValidator } from '../../../middlewares/validators' +import { UserModel } from '../../../models/account/user' +import { VideoModel } from '../../../models/video/video' +import { buildNSFWFilter } from '../../../helpers/express-utils' +import { VideoFilter } from '../../../../shared/models/videos/video-query.type' +import { ActorFollowModel } from '../../../models/activitypub/actor-follow' +import { JobQueue } from '../../../lib/job-queue' +import { logger } from '../../../helpers/logger' + +const mySubscriptionsRouter = express.Router() + +mySubscriptionsRouter.get('/me/subscriptions/videos', + authenticate, + paginationValidator, + videosSortValidator, + setDefaultSort, + setDefaultPagination, + commonVideosFiltersValidator, + asyncMiddleware(getUserSubscriptionVideos) +) + +mySubscriptionsRouter.get('/me/subscriptions/exist', + authenticate, + areSubscriptionsExistValidator, + asyncMiddleware(areSubscriptionsExist) +) + +mySubscriptionsRouter.get('/me/subscriptions', + authenticate, + paginationValidator, + userSubscriptionsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(getUserSubscriptions) +) + +mySubscriptionsRouter.post('/me/subscriptions', + authenticate, + userSubscriptionAddValidator, + asyncMiddleware(addUserSubscription) +) + +mySubscriptionsRouter.get('/me/subscriptions/:uri', + authenticate, + userSubscriptionGetValidator, + getUserSubscription +) + +mySubscriptionsRouter.delete('/me/subscriptions/:uri', + authenticate, + userSubscriptionGetValidator, + asyncRetryTransactionMiddleware(deleteUserSubscription) +) + +// --------------------------------------------------------------------------- + +export { + mySubscriptionsRouter +} + +// --------------------------------------------------------------------------- + +async function areSubscriptionsExist (req: express.Request, res: express.Response) { + const uris = req.query.uris as string[] + const user = res.locals.oauth.token.User as UserModel + + const handles = uris.map(u => { + let [ name, host ] = u.split('@') + if (host === CONFIG.WEBSERVER.HOST) host = null + + return { name, host, uri: u } + }) + + const results = await ActorFollowModel.listSubscribedIn(user.Account.Actor.id, handles) + + const existObject: { [id: string ]: boolean } = {} + for (const handle of handles) { + const obj = results.find(r => { + const server = r.ActorFollowing.Server + + return r.ActorFollowing.preferredUsername === handle.name && + ( + (!server && !handle.host) || + (server.host === handle.host) + ) + }) + + existObject[handle.uri] = obj !== undefined + } + + return res.json(existObject) +} + +async function addUserSubscription (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User as UserModel + const [ name, host ] = req.body.uri.split('@') + + const payload = { + name, + host, + followerActorId: user.Account.Actor.id + } + + JobQueue.Instance.createJob({ type: 'activitypub-follow', payload }) + .catch(err => logger.error('Cannot create follow job for subscription %s.', req.body.uri, err)) + + return res.status(204).end() +} + +function getUserSubscription (req: express.Request, res: express.Response) { + const subscription: ActorFollowModel = res.locals.subscription + + return res.json(subscription.ActorFollowing.VideoChannel.toFormattedJSON()) +} + +async function deleteUserSubscription (req: express.Request, res: express.Response) { + const subscription: ActorFollowModel = res.locals.subscription + + await sequelizeTypescript.transaction(async t => { + return subscription.destroy({ transaction: t }) + }) + + return res.type('json').status(204).end() +} + +async function getUserSubscriptions (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User as UserModel + const actorId = user.Account.Actor.id + + const resultList = await ActorFollowModel.listSubscriptionsForApi(actorId, req.query.start, req.query.count, req.query.sort) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function getUserSubscriptionVideos (req: express.Request, res: express.Response, next: express.NextFunction) { + const user = res.locals.oauth.token.User as UserModel + const resultList = await VideoModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + includeLocalVideos: false, + categoryOneOf: req.query.categoryOneOf, + licenceOneOf: req.query.licenceOneOf, + languageOneOf: req.query.languageOneOf, + tagsOneOf: req.query.tagsOneOf, + tagsAllOf: req.query.tagsAllOf, + nsfw: buildNSFWFilter(res, req.query.nsfw), + filter: req.query.filter as VideoFilter, + withFiles: false, + followerActorId: user.Account.Actor.id, + user + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} -- cgit v1.2.3 From bb8f7872f5a473c47a688b0c282ff34cd78a9835 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 11:01:40 +0100 Subject: Fix peertube CLI documentation --- server/tools/peertube-import-videos.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'server') diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts index 2874a2131..f50aafc35 100644 --- a/server/tools/peertube-import-videos.ts +++ b/server/tools/peertube-import-videos.ts @@ -78,7 +78,11 @@ getSettings() password: program['password'] } - run(user, program['url']).catch(err => console.error(err)) + run(user, program['url']) + .catch(err => { + console.error(err) + process.exit(-1) + }) }) async function promptPassword () { @@ -112,8 +116,12 @@ async function run (user, url: string) { secret: res.body.client_secret } - const res2 = await login(url, client, user) - accessToken = res2.body.access_token + try { + const res = await login(program[ 'url' ], client, user) + accessToken = res.body.access_token + } catch (err) { + throw new Error('Cannot authenticate. Please check your username/password.') + } const youtubeDL = await safeGetYoutubeDL() -- cgit v1.2.3 From 744d0eca195bce7dafeb4a958d0eb3c0046be32d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 11:30:15 +0100 Subject: Refresh remote actors on GET enpoints --- server/controllers/api/accounts.ts | 7 ++ server/controllers/api/video-channel.ts | 6 ++ server/controllers/api/videos/index.ts | 2 +- server/lib/activitypub/actor.ts | 111 +++++++++++---------- server/lib/activitypub/videos.ts | 2 +- .../job-queue/handlers/activitypub-refresher.ts | 25 +++-- server/models/account/account.ts | 4 + server/models/video/video-channel.ts | 4 + 8 files changed, 99 insertions(+), 62 deletions(-) (limited to 'server') diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts index a69a83acf..8c0237203 100644 --- a/server/controllers/api/accounts.ts +++ b/server/controllers/api/accounts.ts @@ -14,6 +14,8 @@ import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' import { VideoChannelModel } from '../../models/video/video-channel' +import { JobQueue } from '../../lib/job-queue' +import { logger } from '../../helpers/logger' const accountsRouter = express.Router() @@ -57,6 +59,11 @@ export { function getAccount (req: express.Request, res: express.Response, next: express.NextFunction) { const account: AccountModel = res.locals.account + if (account.isOutdated()) { + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'actor', url: account.Actor.url } }) + .catch(err => logger.error('Cannot create AP refresher job for actor %s.', account.Actor.url, { err })) + } + return res.json(account.toFormattedJSON()) } diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 3d6a6af7f..db7602139 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -30,6 +30,7 @@ import { updateActorAvatarFile } from '../../lib/avatar' import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' import { resetSequelizeInstance } from '../../helpers/database-utils' import { UserModel } from '../../models/account/user' +import { JobQueue } from '../../lib/job-queue' const auditLogger = auditLoggerFactory('channels') const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) @@ -197,6 +198,11 @@ async function removeVideoChannel (req: express.Request, res: express.Response) async function getVideoChannel (req: express.Request, res: express.Response, next: express.NextFunction) { const videoChannelWithVideos = await VideoChannelModel.loadAndPopulateAccountAndVideos(res.locals.videoChannel.id) + if (videoChannelWithVideos.isOutdated()) { + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'actor', url: videoChannelWithVideos.Actor.url } }) + .catch(err => logger.error('Cannot create AP refresher job for actor %s.', videoChannelWithVideos.Actor.url, { err })) + } + return res.json(videoChannelWithVideos.toFormattedJSON()) } diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 28ac26598..2b2dfa7ca 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -399,7 +399,7 @@ function getVideo (req: express.Request, res: express.Response) { const videoInstance = res.locals.video if (videoInstance.isOutdated()) { - JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoInstance.url } }) + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) } diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f80296725..d728c81d1 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -201,6 +201,62 @@ async function addFetchOutboxJob (actor: ActorModel) { return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) } +async function refreshActorIfNeeded ( + actorArg: ActorModel, + fetchedType: ActorFetchByUrlType +): Promise<{ actor: ActorModel, refreshed: boolean }> { + if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } + + // We need more attributes + const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) + + try { + const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + const { result, statusCode } = await fetchRemoteActor(actorUrl) + + if (statusCode === 404) { + logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) + actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() + return { actor: undefined, refreshed: false } + } + + if (result === undefined) { + logger.warn('Cannot fetch remote actor in refresh actor.') + return { actor, refreshed: false } + } + + return sequelizeTypescript.transaction(async t => { + updateInstanceWithAnother(actor, result.actor) + + if (result.avatarName !== undefined) { + await updateActorAvatarInstance(actor, result.avatarName, t) + } + + // Force update + actor.setDataValue('updatedAt', new Date()) + await actor.save({ transaction: t }) + + if (actor.Account) { + actor.Account.set('name', result.name) + actor.Account.set('description', result.summary) + + await actor.Account.save({ transaction: t }) + } else if (actor.VideoChannel) { + actor.VideoChannel.set('name', result.name) + actor.VideoChannel.set('description', result.summary) + actor.VideoChannel.set('support', result.support) + + await actor.VideoChannel.save({ transaction: t }) + } + + return { refreshed: true, actor } + }) + } catch (err) { + logger.warn('Cannot refresh actor.', { err }) + return { actor, refreshed: false } + } +} + export { getOrCreateActorAndServerAndModel, buildActorInstance, @@ -208,6 +264,7 @@ export { fetchActorTotalItems, fetchAvatarIfExists, updateActorInstance, + refreshActorIfNeeded, updateActorAvatarInstance, addFetchOutboxJob } @@ -373,58 +430,4 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannelCreated } -async function refreshActorIfNeeded ( - actorArg: ActorModel, - fetchedType: ActorFetchByUrlType -): Promise<{ actor: ActorModel, refreshed: boolean }> { - if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } - - // We need more attributes - const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) - - try { - const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) - const { result, statusCode } = await fetchRemoteActor(actorUrl) - - if (statusCode === 404) { - logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) - actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() - return { actor: undefined, refreshed: false } - } - - if (result === undefined) { - logger.warn('Cannot fetch remote actor in refresh actor.') - return { actor, refreshed: false } - } - return sequelizeTypescript.transaction(async t => { - updateInstanceWithAnother(actor, result.actor) - - if (result.avatarName !== undefined) { - await updateActorAvatarInstance(actor, result.avatarName, t) - } - - // Force update - actor.setDataValue('updatedAt', new Date()) - await actor.save({ transaction: t }) - - if (actor.Account) { - actor.Account.set('name', result.name) - actor.Account.set('description', result.summary) - - await actor.Account.save({ transaction: t }) - } else if (actor.VideoChannel) { - actor.VideoChannel.set('name', result.name) - actor.VideoChannel.set('description', result.summary) - actor.VideoChannel.set('support', result.support) - - await actor.VideoChannel.save({ transaction: t }) - } - - return { refreshed: true, actor } - }) - } catch (err) { - logger.warn('Cannot refresh actor.', { err }) - return { actor, refreshed: false } - } -} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 893768769..cbdd981c5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -179,7 +179,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { } if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) - else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) + else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } }) } return { video: videoFromDatabase, created: false } diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 671b0f487..454b975fe 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -1,30 +1,33 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { fetchVideoByUrl } from '../../../helpers/video' -import { refreshVideoIfNeeded } from '../../activitypub' +import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' +import { ActorModel } from '../../../models/activitypub/actor' export type RefreshPayload = { - videoUrl: string - type: 'video' + type: 'video' | 'actor' + url: string } async function refreshAPObject (job: Bull.Job) { const payload = job.data as RefreshPayload - logger.info('Processing AP refresher in job %d for video %s.', job.id, payload.videoUrl) + logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) - if (payload.type === 'video') return refreshAPVideo(payload.videoUrl) + if (payload.type === 'video') return refreshVideo(payload.url) + if (payload.type === 'actor') return refreshActor(payload.url) } // --------------------------------------------------------------------------- export { + refreshActor, refreshAPObject } // --------------------------------------------------------------------------- -async function refreshAPVideo (videoUrl: string) { +async function refreshVideo (videoUrl: string) { const fetchType = 'all' as 'all' const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } @@ -39,3 +42,13 @@ async function refreshAPVideo (videoUrl: string) { await refreshVideoIfNeeded(refreshOptions) } } + +async function refreshActor (actorUrl: string) { + const fetchType = 'all' as 'all' + const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) + + if (actor) { + await refreshActorIfNeeded(actor, fetchType) + } + +} diff --git a/server/models/account/account.ts b/server/models/account/account.ts index a99e9b1ad..84ef0b30d 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -288,6 +288,10 @@ export class AccountModel extends Model { return this.Actor.isOwned() } + isOutdated () { + return this.Actor.isOutdated() + } + getDisplayName () { return this.name } diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 86bf0461a..5598d80f6 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -470,4 +470,8 @@ export class VideoChannelModel extends Model { getDisplayName () { return this.name } + + isOutdated () { + return this.Actor.isOutdated() + } } -- cgit v1.2.3 From 699b059e2d6cdd09685a69261f2ca5cf63053a71 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 12:11:06 +0100 Subject: Fix deleting not found remote actors --- server/lib/activitypub/actor.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'server') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index d728c81d1..edf38bc0a 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -211,7 +211,14 @@ async function refreshActorIfNeeded ( const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) try { - const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + let actorUrl: string + try { + actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + } catch (err) { + logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) + actorUrl = actor.url + } + const { result, statusCode } = await fetchRemoteActor(actorUrl) if (statusCode === 404) { @@ -429,5 +436,3 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannelCreated } - - -- cgit v1.2.3 From 1506307f2f903ce0f80155072a33345c702b7c76 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 16:48:38 +0100 Subject: Increase abuse length to 3000 And correctly handle new lines --- server/initializers/constants.ts | 6 ++-- .../migrations/0325-video-abuse-fields.ts | 37 ++++++++++++++++++++++ server/models/video/video-abuse.ts | 19 ++--------- server/tests/api/check-params/video-abuses.ts | 6 ++-- 4 files changed, 46 insertions(+), 22 deletions(-) create mode 100644 server/initializers/migrations/0325-video-abuse-fields.ts (limited to 'server') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index b18884eeb..93fdd3f03 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -16,7 +16,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 320 +const LAST_MIGRATION_VERSION = 325 // --------------------------------------------------------------------------- @@ -316,8 +316,8 @@ let CONSTRAINTS_FIELDS = { BLOCKED_REASON: { min: 3, max: 250 } // Length }, VIDEO_ABUSES: { - REASON: { min: 2, max: 300 }, // Length - MODERATION_COMMENT: { min: 2, max: 300 } // Length + REASON: { min: 2, max: 3000 }, // Length + MODERATION_COMMENT: { min: 2, max: 3000 } // Length }, VIDEO_BLACKLIST: { REASON: { min: 2, max: 300 } // Length diff --git a/server/initializers/migrations/0325-video-abuse-fields.ts b/server/initializers/migrations/0325-video-abuse-fields.ts new file mode 100644 index 000000000..fca6d666f --- /dev/null +++ b/server/initializers/migrations/0325-video-abuse-fields.ts @@ -0,0 +1,37 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize +}): Promise { + + { + const data = { + type: Sequelize.STRING(3000), + allowNull: false, + defaultValue: null + } + + await utils.queryInterface.changeColumn('videoAbuse', 'reason', data) + } + + { + const data = { + type: Sequelize.STRING(3000), + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.changeColumn('videoAbuse', 'moderationComment', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/models/video/video-abuse.ts b/server/models/video/video-abuse.ts index 4c9e2d05e..cc47644f2 100644 --- a/server/models/video/video-abuse.ts +++ b/server/models/video/video-abuse.ts @@ -1,17 +1,4 @@ -import { - AfterCreate, - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { VideoAbuseObject } from '../../../shared/models/activitypub/objects' import { VideoAbuse } from '../../../shared/models/videos' import { @@ -19,7 +6,6 @@ import { isVideoAbuseReasonValid, isVideoAbuseStateValid } from '../../helpers/custom-validators/video-abuses' -import { Emailer } from '../../lib/emailer' import { AccountModel } from '../account/account' import { getSort, throwIfNotValid } from '../utils' import { VideoModel } from './video' @@ -40,8 +26,9 @@ import { CONSTRAINTS_FIELDS, VIDEO_ABUSE_STATES } from '../../initializers' export class VideoAbuseModel extends Model { @AllowNull(false) + @Default(null) @Is('VideoAbuseReason', value => throwIfNotValid(value, isVideoAbuseReasonValid, 'reason')) - @Column + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_ABUSES.REASON.max)) reason: string @AllowNull(false) diff --git a/server/tests/api/check-params/video-abuses.ts b/server/tests/api/check-params/video-abuses.ts index a79ab4201..3b8f5f14d 100644 --- a/server/tests/api/check-params/video-abuses.ts +++ b/server/tests/api/check-params/video-abuses.ts @@ -113,8 +113,8 @@ describe('Test video abuses API validators', function () { await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) }) - it('Should fail with a reason too big', async function () { - const fields = { reason: 'super'.repeat(61) } + it('Should fail with a too big reason', async function () { + const fields = { reason: 'super'.repeat(605) } await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) }) @@ -154,7 +154,7 @@ describe('Test video abuses API validators', function () { }) it('Should fail with a bad moderation comment', async function () { - const body = { moderationComment: 'b'.repeat(305) } + const body = { moderationComment: 'b'.repeat(3001) } await updateVideoAbuse(server.url, server.accessToken, server.video.uuid, videoAbuseId, body, 400) }) -- cgit v1.2.3 From 44b9c0ba31c4a97e3d874f33226ad935c3a90dd5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 15 Jan 2019 09:45:54 +0100 Subject: Add totalLocalVideoFilesSize in stats --- server/controllers/api/server/stats.ts | 7 +++++-- server/models/redundancy/video-redundancy.ts | 2 +- server/models/video/video-file.ts | 20 ++++++++++++++++++++ server/tests/api/server/stats.ts | 3 ++- 4 files changed, 28 insertions(+), 4 deletions(-) (limited to 'server') diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 85803f69e..89ffd1717 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts @@ -8,6 +8,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' import { cacheRoute } from '../../../middlewares/cache' +import { VideoFileModel } from '../../../models/video/video-file' const statsRouter = express.Router() @@ -16,11 +17,12 @@ statsRouter.get('/stats', asyncMiddleware(getStats) ) -async function getStats (req: express.Request, res: express.Response, next: express.NextFunction) { +async function getStats (req: express.Request, res: express.Response) { const { totalLocalVideos, totalLocalVideoViews, totalVideos } = await VideoModel.getStats() const { totalLocalVideoComments, totalVideoComments } = await VideoCommentModel.getStats() const { totalUsers } = await UserModel.getStats() const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() + const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() const videosRedundancyStats = await Promise.all( CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { @@ -32,8 +34,9 @@ async function getStats (req: express.Request, res: express.Response, next: expr const data: ServerStats = { totalLocalVideos, totalLocalVideoViews, - totalVideos, + totalLocalVideoFilesSize, totalLocalVideoComments, + totalVideos, totalVideoComments, totalUsers, totalInstanceFollowers, diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 8b6cd146a..8f2ef2d9a 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -395,7 +395,7 @@ export class VideoRedundancyModel extends Model { ] } - return VideoRedundancyModel.find(query as any) // FIXME: typings + return VideoRedundancyModel.findOne(query as any) // FIXME: typings .then((r: any) => ({ totalUsed: parseInt(r.totalUsed.toString(), 10), totalVideos: r.totalVideos, diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 0fd868cd6..1f1b76c1e 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -120,6 +120,26 @@ export class VideoFileModel extends Model { return VideoFileModel.findById(id, options) } + static async getStats () { + let totalLocalVideoFilesSize = await VideoFileModel.sum('size', { + include: [ + { + attributes: [], + model: VideoModel.unscoped(), + where: { + remote: false + } + } + ] + } as any) + // Sequelize could return null... + if (!totalLocalVideoFilesSize) totalLocalVideoFilesSize = 0 + + return { + totalLocalVideoFilesSize + } + } + hasSameUniqueKeysThan (other: VideoFileModel) { return this.fps === other.fps && this.resolution === other.resolution && diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index 517b4e542..9858e2b15 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts @@ -39,7 +39,7 @@ describe('Test stats (excluding redundancy)', function () { } await createUser(servers[0].url, servers[0].accessToken, user.username, user.password) - const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, {}) + const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, { fixture: 'video_short.webm' }) const videoUUID = resVideo.body.video.uuid await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment') @@ -60,6 +60,7 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalLocalVideoComments).to.equal(1) expect(data.totalLocalVideos).to.equal(1) expect(data.totalLocalVideoViews).to.equal(1) + expect(data.totalLocalVideoFilesSize).to.equal(218910) expect(data.totalUsers).to.equal(2) expect(data.totalVideoComments).to.equal(1) expect(data.totalVideos).to.equal(1) -- cgit v1.2.3 From 848f499def54db2dd36437ef0dfb74dd5041c23b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 15 Jan 2019 11:14:12 +0100 Subject: Prepare Dislike/Flag/View fixes For now we Create these activities, but we should just send them directly. This fix handles correctly direct Dislikes/Flags/Views, we'll implement the sending correctly these activities in the next peertube version --- server/helpers/activitypub.ts | 4 +- .../custom-validators/activitypub/activity.ts | 99 +++-- .../helpers/custom-validators/activitypub/actor.ts | 25 +- .../custom-validators/activitypub/announce.ts | 13 - .../custom-validators/activitypub/cache-file.ts | 16 +- .../helpers/custom-validators/activitypub/flag.ts | 14 + .../helpers/custom-validators/activitypub/misc.ts | 24 +- .../helpers/custom-validators/activitypub/rate.ts | 15 +- .../helpers/custom-validators/activitypub/undo.ts | 20 - .../activitypub/video-comments.ts | 11 - .../custom-validators/activitypub/videos.ts | 19 - .../helpers/custom-validators/activitypub/view.ts | 10 +- server/lib/activitypub/actor.ts | 4 +- server/lib/activitypub/process/process-accept.ts | 1 - server/lib/activitypub/process/process-create.ts | 118 ++--- server/lib/activitypub/process/process-dislike.ts | 52 +++ server/lib/activitypub/process/process-flag.ts | 49 +++ server/lib/activitypub/process/process-follow.ts | 3 +- server/lib/activitypub/process/process-like.ts | 3 +- server/lib/activitypub/process/process-undo.ts | 8 +- server/lib/activitypub/process/process-view.ts | 35 ++ server/lib/activitypub/process/process.ts | 12 +- server/lib/activitypub/share.ts | 4 +- server/lib/activitypub/video-rates.ts | 4 +- server/lib/activitypub/videos.ts | 6 +- server/tests/api/check-params/contact-form.ts | 4 + server/tests/api/server/redundancy.ts | 479 --------------------- server/tests/api/server/stats.ts | 1 + 28 files changed, 303 insertions(+), 750 deletions(-) delete mode 100644 server/helpers/custom-validators/activitypub/announce.ts create mode 100644 server/helpers/custom-validators/activitypub/flag.ts delete mode 100644 server/helpers/custom-validators/activitypub/undo.ts create mode 100644 server/lib/activitypub/process/process-dislike.ts create mode 100644 server/lib/activitypub/process/process-flag.ts create mode 100644 server/lib/activitypub/process/process-view.ts delete mode 100644 server/tests/api/server/redundancy.ts (limited to 'server') diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 79b76fa0b..f1430055f 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -106,7 +106,7 @@ function buildSignedActivity (byActor: ActorModel, data: Object) { return signJsonLDObject(byActor, activity) as Promise } -function getAPUrl (activity: string | { id: string }) { +function getAPId (activity: string | { id: string }) { if (typeof activity === 'string') return activity return activity.id @@ -123,7 +123,7 @@ function checkUrlsSameHost (url1: string, url2: string) { export { checkUrlsSameHost, - getAPUrl, + getAPId, activityPubContextify, activityPubCollectionPagination, buildSignedActivity diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 2562ead9b..b24590d9d 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -1,26 +1,14 @@ import * as validator from 'validator' import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { - isActorAcceptActivityValid, - isActorDeleteActivityValid, - isActorFollowActivityValid, - isActorRejectActivityValid, - isActorUpdateActivityValid -} from './actor' -import { isAnnounceActivityValid } from './announce' -import { isActivityPubUrlValid } from './misc' -import { isDislikeActivityValid, isLikeActivityValid } from './rate' -import { isUndoActivityValid } from './undo' -import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments' -import { - isVideoFlagValid, - isVideoTorrentDeleteActivityValid, - sanitizeAndCheckVideoTorrentCreateActivity, - sanitizeAndCheckVideoTorrentUpdateActivity -} from './videos' +import { sanitizeAndCheckActorObject } from './actor' +import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' +import { isDislikeActivityValid } from './rate' +import { sanitizeAndCheckVideoCommentObject } from './video-comments' +import { sanitizeAndCheckVideoTorrentObject } from './videos' import { isViewActivityValid } from './view' import { exists } from '../misc' -import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file' +import { isCacheFileObjectValid } from './cache-file' +import { isFlagActivityValid } from './flag' function isRootActivityValid (activity: any) { return Array.isArray(activity['@context']) && ( @@ -46,7 +34,10 @@ const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean Reject: checkRejectActivity, Announce: checkAnnounceActivity, Undo: checkUndoActivity, - Like: checkLikeActivity + Like: checkLikeActivity, + View: checkViewActivity, + Flag: checkFlagActivity, + Dislike: checkDislikeActivity } function isActivityValid (activity: any) { @@ -66,47 +57,79 @@ export { // --------------------------------------------------------------------------- +function checkViewActivity (activity: any) { + return isBaseActivityValid(activity, 'View') && + isViewActivityValid(activity) +} + +function checkFlagActivity (activity: any) { + return isBaseActivityValid(activity, 'Flag') && + isFlagActivityValid(activity) +} + +function checkDislikeActivity (activity: any) { + return isBaseActivityValid(activity, 'Dislike') && + isDislikeActivityValid(activity) +} + function checkCreateActivity (activity: any) { - return isViewActivityValid(activity) || - isDislikeActivityValid(activity) || - sanitizeAndCheckVideoTorrentCreateActivity(activity) || - isVideoFlagValid(activity) || - isVideoCommentCreateActivityValid(activity) || - isCacheFileCreateActivityValid(activity) + return isBaseActivityValid(activity, 'Create') && + ( + isViewActivityValid(activity.object) || + isDislikeActivityValid(activity.object) || + isFlagActivityValid(activity.object) || + + isCacheFileObjectValid(activity.object) || + sanitizeAndCheckVideoCommentObject(activity.object) || + sanitizeAndCheckVideoTorrentObject(activity.object) + ) } function checkUpdateActivity (activity: any) { - return isCacheFileUpdateActivityValid(activity) || - sanitizeAndCheckVideoTorrentUpdateActivity(activity) || - isActorUpdateActivityValid(activity) + return isBaseActivityValid(activity, 'Update') && + ( + isCacheFileObjectValid(activity.object) || + sanitizeAndCheckVideoTorrentObject(activity.object) || + sanitizeAndCheckActorObject(activity.object) + ) } function checkDeleteActivity (activity: any) { - return isVideoTorrentDeleteActivityValid(activity) || - isActorDeleteActivityValid(activity) || - isVideoCommentDeleteActivityValid(activity) + // We don't really check objects + return isBaseActivityValid(activity, 'Delete') && + isObjectValid(activity.object) } function checkFollowActivity (activity: any) { - return isActorFollowActivityValid(activity) + return isBaseActivityValid(activity, 'Follow') && + isObjectValid(activity.object) } function checkAcceptActivity (activity: any) { - return isActorAcceptActivityValid(activity) + return isBaseActivityValid(activity, 'Accept') } function checkRejectActivity (activity: any) { - return isActorRejectActivityValid(activity) + return isBaseActivityValid(activity, 'Reject') } function checkAnnounceActivity (activity: any) { - return isAnnounceActivityValid(activity) + return isBaseActivityValid(activity, 'Announce') && + isObjectValid(activity.object) } function checkUndoActivity (activity: any) { - return isUndoActivityValid(activity) + return isBaseActivityValid(activity, 'Undo') && + ( + checkFollowActivity(activity.object) || + checkLikeActivity(activity.object) || + checkDislikeActivity(activity.object) || + checkAnnounceActivity(activity.object) || + checkCreateActivity(activity.object) + ) } function checkLikeActivity (activity: any) { - return isLikeActivityValid(activity) + return isBaseActivityValid(activity, 'Like') && + isObjectValid(activity.object) } diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts index 070632a20..c05f60f14 100644 --- a/server/helpers/custom-validators/activitypub/actor.ts +++ b/server/helpers/custom-validators/activitypub/actor.ts @@ -73,24 +73,10 @@ function isActorDeleteActivityValid (activity: any) { return isBaseActivityValid(activity, 'Delete') } -function isActorFollowActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Follow') && - isActivityPubUrlValid(activity.object) -} - -function isActorAcceptActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Accept') -} - -function isActorRejectActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Reject') -} - -function isActorUpdateActivityValid (activity: any) { - normalizeActor(activity.object) +function sanitizeAndCheckActorObject (object: any) { + normalizeActor(object) - return isBaseActivityValid(activity, 'Update') && - isActorObjectValid(activity.object) + return isActorObjectValid(object) } function normalizeActor (actor: any) { @@ -139,10 +125,7 @@ export { isActorObjectValid, isActorFollowingCountValid, isActorFollowersCountValid, - isActorFollowActivityValid, - isActorAcceptActivityValid, - isActorRejectActivityValid, isActorDeleteActivityValid, - isActorUpdateActivityValid, + sanitizeAndCheckActorObject, isValidActorHandle } diff --git a/server/helpers/custom-validators/activitypub/announce.ts b/server/helpers/custom-validators/activitypub/announce.ts deleted file mode 100644 index 0519c6026..000000000 --- a/server/helpers/custom-validators/activitypub/announce.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { isActivityPubUrlValid, isBaseActivityValid } from './misc' - -function isAnnounceActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Announce') && - ( - isActivityPubUrlValid(activity.object) || - (activity.object && isActivityPubUrlValid(activity.object.id)) - ) -} - -export { - isAnnounceActivityValid -} diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts index bd70934c8..e2bd0c55e 100644 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ b/server/helpers/custom-validators/activitypub/cache-file.ts @@ -1,18 +1,8 @@ -import { isActivityPubUrlValid, isBaseActivityValid } from './misc' +import { isActivityPubUrlValid } from './misc' import { isRemoteVideoUrlValid } from './videos' -import { isDateValid, exists } from '../misc' +import { exists, isDateValid } from '../misc' import { CacheFileObject } from '../../../../shared/models/activitypub/objects' -function isCacheFileCreateActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Create') && - isCacheFileObjectValid(activity.object) -} - -function isCacheFileUpdateActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Update') && - isCacheFileObjectValid(activity.object) -} - function isCacheFileObjectValid (object: CacheFileObject) { return exists(object) && object.type === 'CacheFile' && @@ -22,7 +12,5 @@ function isCacheFileObjectValid (object: CacheFileObject) { } export { - isCacheFileUpdateActivityValid, - isCacheFileCreateActivityValid, isCacheFileObjectValid } diff --git a/server/helpers/custom-validators/activitypub/flag.ts b/server/helpers/custom-validators/activitypub/flag.ts new file mode 100644 index 000000000..6452e297c --- /dev/null +++ b/server/helpers/custom-validators/activitypub/flag.ts @@ -0,0 +1,14 @@ +import { isActivityPubUrlValid } from './misc' +import { isVideoAbuseReasonValid } from '../video-abuses' + +function isFlagActivityValid (activity: any) { + return activity.type === 'Flag' && + isVideoAbuseReasonValid(activity.content) && + isActivityPubUrlValid(activity.object) +} + +// --------------------------------------------------------------------------- + +export { + isFlagActivityValid +} diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 4e2c57f04..f1762d11c 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -28,15 +28,20 @@ function isBaseActivityValid (activity: any, type: string) { return (activity['@context'] === undefined || Array.isArray(activity['@context'])) && activity.type === type && isActivityPubUrlValid(activity.id) && - exists(activity.actor) && - (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) && - ( - activity.to === undefined || - (Array.isArray(activity.to) && activity.to.every(t => isActivityPubUrlValid(t))) - ) && + isObjectValid(activity.actor) && + isUrlCollectionValid(activity.to) && + isUrlCollectionValid(activity.cc) +} + +function isUrlCollectionValid (collection: any) { + return collection === undefined || + (Array.isArray(collection) && collection.every(t => isActivityPubUrlValid(t))) +} + +function isObjectValid (object: any) { + return exists(object) && ( - activity.cc === undefined || - (Array.isArray(activity.cc) && activity.cc.every(t => isActivityPubUrlValid(t))) + isActivityPubUrlValid(object) || isActivityPubUrlValid(object.id) ) } @@ -57,5 +62,6 @@ export { isUrlValid, isActivityPubUrlValid, isBaseActivityValid, - setValidAttributedTo + setValidAttributedTo, + isObjectValid } diff --git a/server/helpers/custom-validators/activitypub/rate.ts b/server/helpers/custom-validators/activitypub/rate.ts index e70bd94b8..ba68e8074 100644 --- a/server/helpers/custom-validators/activitypub/rate.ts +++ b/server/helpers/custom-validators/activitypub/rate.ts @@ -1,20 +1,13 @@ -import { isActivityPubUrlValid, isBaseActivityValid } from './misc' - -function isLikeActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Like') && - isActivityPubUrlValid(activity.object) -} +import { isActivityPubUrlValid, isObjectValid } from './misc' function isDislikeActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Create') && - activity.object.type === 'Dislike' && - isActivityPubUrlValid(activity.object.actor) && - isActivityPubUrlValid(activity.object.object) + return activity.type === 'Dislike' && + isActivityPubUrlValid(activity.actor) && + isObjectValid(activity.object) } // --------------------------------------------------------------------------- export { - isLikeActivityValid, isDislikeActivityValid } diff --git a/server/helpers/custom-validators/activitypub/undo.ts b/server/helpers/custom-validators/activitypub/undo.ts deleted file mode 100644 index 578035893..000000000 --- a/server/helpers/custom-validators/activitypub/undo.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { isActorFollowActivityValid } from './actor' -import { isBaseActivityValid } from './misc' -import { isDislikeActivityValid, isLikeActivityValid } from './rate' -import { isAnnounceActivityValid } from './announce' -import { isCacheFileCreateActivityValid } from './cache-file' - -function isUndoActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Undo') && - ( - isActorFollowActivityValid(activity.object) || - isLikeActivityValid(activity.object) || - isDislikeActivityValid(activity.object) || - isAnnounceActivityValid(activity.object) || - isCacheFileCreateActivityValid(activity.object) - ) -} - -export { - isUndoActivityValid -} diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts index 051c4565a..0415db21c 100644 --- a/server/helpers/custom-validators/activitypub/video-comments.ts +++ b/server/helpers/custom-validators/activitypub/video-comments.ts @@ -3,11 +3,6 @@ import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' import { exists, isArray, isDateValid } from '../misc' import { isActivityPubUrlValid, isBaseActivityValid } from './misc' -function isVideoCommentCreateActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Create') && - sanitizeAndCheckVideoCommentObject(activity.object) -} - function sanitizeAndCheckVideoCommentObject (comment: any) { if (!comment || comment.type !== 'Note') return false @@ -25,15 +20,9 @@ function sanitizeAndCheckVideoCommentObject (comment: any) { ) // Only accept public comments } -function isVideoCommentDeleteActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Delete') -} - // --------------------------------------------------------------------------- export { - isVideoCommentCreateActivityValid, - isVideoCommentDeleteActivityValid, sanitizeAndCheckVideoCommentObject } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 95fe824b9..0f34aab21 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -14,27 +14,11 @@ import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from import { VideoState } from '../../../../shared/models/videos' import { isVideoAbuseReasonValid } from '../video-abuses' -function sanitizeAndCheckVideoTorrentCreateActivity (activity: any) { - return isBaseActivityValid(activity, 'Create') && - sanitizeAndCheckVideoTorrentObject(activity.object) -} - function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { return isBaseActivityValid(activity, 'Update') && sanitizeAndCheckVideoTorrentObject(activity.object) } -function isVideoTorrentDeleteActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Delete') -} - -function isVideoFlagValid (activity: any) { - return isBaseActivityValid(activity, 'Create') && - activity.object.type === 'Flag' && - isVideoAbuseReasonValid(activity.object.content) && - isActivityPubUrlValid(activity.object.object) -} - function isActivityPubVideoDurationValid (value: string) { // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration return exists(value) && @@ -103,11 +87,8 @@ function isRemoteVideoUrlValid (url: any) { // --------------------------------------------------------------------------- export { - sanitizeAndCheckVideoTorrentCreateActivity, sanitizeAndCheckVideoTorrentUpdateActivity, - isVideoTorrentDeleteActivityValid, isRemoteStringIdentifierValid, - isVideoFlagValid, sanitizeAndCheckVideoTorrentObject, isRemoteVideoUrlValid } diff --git a/server/helpers/custom-validators/activitypub/view.ts b/server/helpers/custom-validators/activitypub/view.ts index 7a3aca6f5..41d16469f 100644 --- a/server/helpers/custom-validators/activitypub/view.ts +++ b/server/helpers/custom-validators/activitypub/view.ts @@ -1,11 +1,11 @@ -import { isActivityPubUrlValid, isBaseActivityValid } from './misc' +import { isActivityPubUrlValid } from './misc' function isViewActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Create') && - activity.object.type === 'View' && - isActivityPubUrlValid(activity.object.actor) && - isActivityPubUrlValid(activity.object.object) + return activity.type === 'View' && + isActivityPubUrlValid(activity.actor) && + isActivityPubUrlValid(activity.object) } + // --------------------------------------------------------------------------- export { diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index edf38bc0a..8215840da 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -4,7 +4,7 @@ import * as url from 'url' import * as uuidv4 from 'uuid/v4' import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' @@ -42,7 +42,7 @@ async function getOrCreateActorAndServerAndModel ( recurseIfNeeded = true, updateCollections = false ) { - const actorUrl = getAPUrl(activityActor) + const actorUrl = getAPId(activityActor) let created = false let actor = await fetchActorByUrl(actorUrl, fetchType) diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 605705ad3..ebb275e34 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts @@ -2,7 +2,6 @@ import { ActivityAccept } from '../../../../shared/models/activitypub' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { addFetchOutboxJob } from '../actor' -import { Notifier } from '../../notifier' async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) { if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 2e04ee843..5f4d793a5 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -1,36 +1,44 @@ -import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared' -import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' +import { ActivityCreate, CacheFileObject, VideoTorrentObject } from '../../../../shared' import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' -import { VideoAbuseModel } from '../../../models/video/video-abuse' import { addVideoComment, resolveThread } from '../video-comments' import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardVideoRelatedActivity } from '../send/utils' -import { Redis } from '../../redis' import { createOrUpdateCacheFile } from '../cache-file' -import { getVideoDislikeActivityPubUrl } from '../url' import { Notifier } from '../../notifier' +import { processViewActivity } from './process-view' +import { processDislikeActivity } from './process-dislike' +import { processFlagActivity } from './process-flag' async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { const activityObject = activity.object const activityType = activityObject.type if (activityType === 'View') { - return processCreateView(byActor, activity) - } else if (activityType === 'Dislike') { - return retryTransactionWrapper(processCreateDislike, byActor, activity) - } else if (activityType === 'Video') { + return processViewActivity(activity, byActor) + } + + if (activityType === 'Dislike') { + return retryTransactionWrapper(processDislikeActivity, activity, byActor) + } + + if (activityType === 'Flag') { + return retryTransactionWrapper(processFlagActivity, activity, byActor) + } + + if (activityType === 'Video') { return processCreateVideo(activity) - } else if (activityType === 'Flag') { - return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject) - } else if (activityType === 'Note') { - return retryTransactionWrapper(processCreateVideoComment, byActor, activity) - } else if (activityType === 'CacheFile') { - return retryTransactionWrapper(processCacheFile, byActor, activity) + } + + if (activityType === 'Note') { + return retryTransactionWrapper(processCreateVideoComment, activity, byActor) + } + + if (activityType === 'CacheFile') { + return retryTransactionWrapper(processCacheFile, activity, byActor) } logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) @@ -55,56 +63,7 @@ async function processCreateVideo (activity: ActivityCreate) { return video } -async function processCreateDislike (byActor: ActorModel, activity: ActivityCreate) { - const dislike = activity.object as DislikeObject - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) - - return sequelizeTypescript.transaction(async t => { - const rate = { - type: 'dislike' as 'dislike', - videoId: video.id, - accountId: byAccount.id - } - - const [ , created ] = await AccountVideoRateModel.findOrCreate({ - where: rate, - defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), - transaction: t - }) - if (created === true) await video.increment('dislikes', { transaction: t }) - - if (video.isOwned() && created === true) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, t, exceptions, video) - } - }) -} - -async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { - const view = activity.object as ViewObject - - const options = { - videoObject: view.object, - fetchType: 'only-video' as 'only-video' - } - const { video } = await getOrCreateVideoAndAccountAndChannel(options) - - await Redis.Instance.addVideoView(video.id) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} - -async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { +async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) { const cacheFile = activity.object as CacheFileObject const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) @@ -120,32 +79,7 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) } } -async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { - logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) - - const account = byActor.Account - if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) - - return sequelizeTypescript.transaction(async t => { - const videoAbuseData = { - reporterAccountId: account.id, - reason: videoAbuseToCreateData.content, - videoId: video.id, - state: VideoAbuseState.PENDING - } - - const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) - videoAbuseInstance.Video = video - - Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) - - logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) - }) -} - -async function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreate) { +async function processCreateVideoComment (activity: ActivityCreate, byActor: ActorModel) { const commentObject = activity.object as VideoCommentObject const byAccount = byActor.Account diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts new file mode 100644 index 000000000..bfd69e07a --- /dev/null +++ b/server/lib/activitypub/process/process-dislike.ts @@ -0,0 +1,52 @@ +import { ActivityCreate, ActivityDislike } from '../../../../shared' +import { DislikeObject } from '../../../../shared/models/activitypub/objects' +import { retryTransactionWrapper } from '../../../helpers/database-utils' +import { sequelizeTypescript } from '../../../initializers' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate' +import { ActorModel } from '../../../models/activitypub/actor' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { forwardVideoRelatedActivity } from '../send/utils' +import { getVideoDislikeActivityPubUrl } from '../url' + +async function processDislikeActivity (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { + return retryTransactionWrapper(processDislike, activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processDislikeActivity +} + +// --------------------------------------------------------------------------- + +async function processDislike (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { + const dislikeObject = activity.type === 'Dislike' ? activity.object : (activity.object as DislikeObject).object + const byAccount = byActor.Account + + if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) + + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) + + return sequelizeTypescript.transaction(async t => { + const rate = { + type: 'dislike' as 'dislike', + videoId: video.id, + accountId: byAccount.id + } + + const [ , created ] = await AccountVideoRateModel.findOrCreate({ + where: rate, + defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), + transaction: t + }) + if (created === true) await video.increment('dislikes', { transaction: t }) + + if (video.isOwned() && created === true) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + + await forwardVideoRelatedActivity(activity, t, exceptions, video) + } + }) +} diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts new file mode 100644 index 000000000..79ce6fb41 --- /dev/null +++ b/server/lib/activitypub/process/process-flag.ts @@ -0,0 +1,49 @@ +import { ActivityCreate, ActivityFlag, VideoAbuseState } from '../../../../shared' +import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects' +import { retryTransactionWrapper } from '../../../helpers/database-utils' +import { logger } from '../../../helpers/logger' +import { sequelizeTypescript } from '../../../initializers' +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoAbuseModel } from '../../../models/video/video-abuse' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { Notifier } from '../../notifier' +import { getAPId } from '../../../helpers/activitypub' + +async function processFlagActivity (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { + return retryTransactionWrapper(processCreateVideoAbuse, activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processFlagActivity +} + +// --------------------------------------------------------------------------- + +async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { + const flag = activity.type === 'Flag' ? activity : (activity.object as VideoAbuseObject) + + logger.debug('Reporting remote abuse for video %s.', getAPId(flag.object)) + + const account = byActor.Account + if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) + + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object }) + + return sequelizeTypescript.transaction(async t => { + const videoAbuseData = { + reporterAccountId: account.id, + reason: flag.content, + videoId: video.id, + state: VideoAbuseState.PENDING + } + + const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) + videoAbuseInstance.Video = video + + Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) + + logger.info('Remote abuse for video uuid %s created', flag.object) + }) +} diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index a67892440..0cd537187 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts @@ -6,9 +6,10 @@ import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { sendAccept } from '../send' import { Notifier } from '../../notifier' +import { getAPId } from '../../../helpers/activitypub' async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { - const activityObject = activity.object + const activityObject = getAPId(activity.object) return retryTransactionWrapper(processFollow, byActor, activityObject) } diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index e8e97eece..2a04167d7 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -6,6 +6,7 @@ import { ActorModel } from '../../../models/activitypub/actor' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { getVideoLikeActivityPubUrl } from '../url' +import { getAPId } from '../../../helpers/activitypub' async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { return retryTransactionWrapper(processLikeVideo, byActor, activity) @@ -20,7 +21,7 @@ export { // --------------------------------------------------------------------------- async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { - const videoUrl = activity.object + const videoUrl = getAPId(activity.object) const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 438a013b6..ed0177a67 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -26,6 +26,10 @@ async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) } } + if (activityToUndo.type === 'Dislike') { + return retryTransactionWrapper(processUndoDislike, byActor, activity) + } + if (activityToUndo.type === 'Follow') { return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) } @@ -72,7 +76,9 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { } async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { - const dislike = activity.object.object as DislikeObject + const dislike = activity.object.type === 'Dislike' + ? activity.object + : activity.object.object as DislikeObject const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts new file mode 100644 index 000000000..8f66d3630 --- /dev/null +++ b/server/lib/activitypub/process/process-view.ts @@ -0,0 +1,35 @@ +import { ActorModel } from '../../../models/activitypub/actor' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' +import { forwardVideoRelatedActivity } from '../send/utils' +import { Redis } from '../../redis' +import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' + +async function processViewActivity (activity: ActivityView | ActivityCreate, byActor: ActorModel) { + return processCreateView(activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processViewActivity +} + +// --------------------------------------------------------------------------- + +async function processCreateView (activity: ActivityView | ActivityCreate, byActor: ActorModel) { + const videoObject = activity.type === 'View' ? activity.object : (activity.object as ViewObject).object + + const options = { + videoObject: videoObject, + fetchType: 'only-video' as 'only-video' + } + const { video } = await getOrCreateVideoAndAccountAndChannel(options) + + await Redis.Instance.addVideoView(video.id) + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + await forwardVideoRelatedActivity(activity, undefined, exceptions, video) + } +} diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index 2479d5da2..9dd241402 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -1,5 +1,5 @@ import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { checkUrlsSameHost, getAPUrl } from '../../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/activitypub/actor' import { processAcceptActivity } from './process-accept' @@ -12,6 +12,9 @@ import { processRejectActivity } from './process-reject' import { processUndoActivity } from './process-undo' import { processUpdateActivity } from './process-update' import { getOrCreateActorAndServerAndModel } from '../actor' +import { processDislikeActivity } from './process-dislike' +import { processFlagActivity } from './process-flag' +import { processViewActivity } from './process-view' const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise } = { Create: processCreateActivity, @@ -22,7 +25,10 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: Ac Reject: processRejectActivity, Announce: processAnnounceActivity, Undo: processUndoActivity, - Like: processLikeActivity + Like: processLikeActivity, + Dislike: processDislikeActivity, + Flag: processFlagActivity, + View: processViewActivity } async function processActivities ( @@ -40,7 +46,7 @@ async function processActivities ( continue } - const actorUrl = getAPUrl(activity.actor) + const actorUrl = getAPId(activity.actor) // When we fetch remote data, we don't have signature if (options.signatureActor && actorUrl !== options.signatureActor.url) { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 170e49238..1767df0ae 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -11,7 +11,7 @@ import { doRequest } from '../../helpers/requests' import { getOrCreateActorAndServerAndModel } from './actor' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -41,7 +41,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') - const actorUrl = getAPUrl(body.actor) + const actorUrl = getAPId(body.actor) if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) } diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 2cce67f0c..45a2b22ea 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' import { doRequest } from '../../helpers/requests' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { ActorModel } from '../../models/activitypub/actor' import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' @@ -26,7 +26,7 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') - const actorUrl = getAPUrl(body.actor) + const actorUrl = getAPId(body.actor) if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index cbdd981c5..e1e523499 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -28,7 +28,7 @@ import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' -import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { Notifier } from '../notifier' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { @@ -155,7 +155,7 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid } async function getOrCreateVideoAndAccountAndChannel (options: { - videoObject: VideoTorrentObject | string, + videoObject: { id: string } | string, syncParam?: SyncParam, fetchType?: VideoFetchByUrlType, allowRefresh?: boolean // true by default @@ -166,7 +166,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { const allowRefresh = options.allowRefresh !== false // Get video url - const videoUrl = getAPUrl(options.videoObject) + const videoUrl = getAPId(options.videoObject) let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) if (videoFromDatabase) { diff --git a/server/tests/api/check-params/contact-form.ts b/server/tests/api/check-params/contact-form.ts index 2407ac0b5..c7e014b1f 100644 --- a/server/tests/api/check-params/contact-form.ts +++ b/server/tests/api/check-params/contact-form.ts @@ -46,6 +46,8 @@ describe('Test contact form API validators', function () { }) it('Should not accept a contact form if it is disabled in the configuration', async function () { + this.timeout(10000) + killallServers([ server ]) // Contact form is disabled @@ -54,6 +56,8 @@ describe('Test contact form API validators', function () { }) it('Should not accept a contact form if from email is invalid', async function () { + this.timeout(10000) + killallServers([ server ]) // Email & contact form enabled diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts deleted file mode 100644 index 8053d0491..000000000 --- a/server/tests/api/server/redundancy.ts +++ /dev/null @@ -1,479 +0,0 @@ -/* tslint:disable:no-unused-expression */ - -import * as chai from 'chai' -import 'mocha' -import { VideoDetails } from '../../../../shared/models/videos' -import { - doubleFollow, - flushAndRunMultipleServers, - getFollowingListPaginationAndSort, - getVideo, - immutableAssign, - killallServers, makeGetRequest, - root, - ServerInfo, - setAccessTokensToServers, unfollow, - uploadVideo, - viewVideo, - wait, - waitUntilLog, - checkVideoFilesWereRemoved, removeVideo -} from '../../../../shared/utils' -import { waitJobs } from '../../../../shared/utils/server/jobs' -import * as magnetUtil from 'magnet-uri' -import { updateRedundancy } from '../../../../shared/utils/server/redundancy' -import { ActorFollow } from '../../../../shared/models/actors' -import { readdir } from 'fs-extra' -import { join } from 'path' -import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' -import { getStats } from '../../../../shared/utils/server/stats' -import { ServerStats } from '../../../../shared/models/server/server-stats.model' - -const expect = chai.expect - -let servers: ServerInfo[] = [] -let video1Server2UUID: string - -function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { - const parsed = magnetUtil.decode(file.magnetUri) - - for (const ws of baseWebseeds) { - const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) - expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined - } - - expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) -} - -async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { - const config = { - redundancy: { - videos: { - check_interval: '5 seconds', - strategies: [ - immutableAssign({ - min_lifetime: '1 hour', - strategy: strategy, - size: '100KB' - }, additionalParams) - ] - } - } - } - servers = await flushAndRunMultipleServers(3, config) - - // Get the access tokens - await setAccessTokensToServers(servers) - - { - const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) - video1Server2UUID = res.body.video.uuid - - await viewVideo(servers[ 1 ].url, video1Server2UUID) - } - - await waitJobs(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[ 0 ], servers[ 1 ]) - // Server 1 and server 3 follow each other - await doubleFollow(servers[ 0 ], servers[ 2 ]) - // Server 2 and server 3 follow each other - await doubleFollow(servers[ 1 ], servers[ 2 ]) - - await waitJobs(servers) -} - -async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID - - const webseeds = [ - 'http://localhost:9002/static/webseed/' + videoUUID - ] - - for (const server of servers) { - { - const res = await getVideo(server.url, videoUUID) - - const video: VideoDetails = res.body - for (const f of video.files) { - checkMagnetWebseeds(f, webseeds, server) - } - } - } -} - -async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - const stat = data.videosRedundancy[0] - - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(102400) - expect(stat.totalUsed).to.be.at.least(1).and.below(102401) - expect(stat.totalVideoFiles).to.equal(4) - expect(stat.totalVideos).to.equal(1) -} - -async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(102400) - expect(stat.totalUsed).to.equal(0) - expect(stat.totalVideoFiles).to.equal(0) - expect(stat.totalVideos).to.equal(0) -} - -async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID - - const webseeds = [ - 'http://localhost:9001/static/webseed/' + videoUUID, - 'http://localhost:9002/static/webseed/' + videoUUID - ] - - for (const server of servers) { - const res = await getVideo(server.url, videoUUID) - - const video: VideoDetails = res.body - - for (const file of video.files) { - checkMagnetWebseeds(file, webseeds, server) - - // Only servers 1 and 2 have the video - if (server.serverNumber !== 3) { - await makeGetRequest({ - url: server.url, - statusCodeExpected: 200, - path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) - } - } - } - - for (const directory of [ 'test1', 'test2' ]) { - const files = await readdir(join(root(), directory, 'videos')) - expect(files).to.have.length.at.least(4) - - for (const resolution of [ 240, 360, 480, 720 ]) { - expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined - } - } -} - -async function enableRedundancyOnServer1 () { - await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) - - const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') - const follows: ActorFollow[] = res.body.data - const server2 = follows.find(f => f.following.host === 'localhost:9002') - const server3 = follows.find(f => f.following.host === 'localhost:9003') - - expect(server3).to.not.be.undefined - expect(server3.following.hostRedundancyAllowed).to.be.false - - expect(server2).to.not.be.undefined - expect(server2.following.hostRedundancyAllowed).to.be.true -} - -async function disableRedundancyOnServer1 () { - await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) - - const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') - const follows: ActorFollow[] = res.body.data - const server2 = follows.find(f => f.following.host === 'localhost:9002') - const server3 = follows.find(f => f.following.host === 'localhost:9003') - - expect(server3).to.not.be.undefined - expect(server3.following.hostRedundancyAllowed).to.be.false - - expect(server2).to.not.be.undefined - expect(server2.following.hostRedundancyAllowed).to.be.false -} - -async function cleanServers () { - killallServers(servers) -} - -describe('Test videos redundancy', function () { - - describe('With most-views strategy', function () { - const strategy = 'most-views' - - before(function () { - this.timeout(120000) - - return runServers(strategy) - }) - - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) - await checkStatsWith1Webseed(strategy) - }) - - it('Should enable redundancy on server 1', function () { - return enableRedundancyOnServer1() - }) - - it('Should have 2 webseed on the first video', async function () { - this.timeout(40000) - - await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) - await waitJobs(servers) - - await check2Webseeds(strategy) - await checkStatsWith2Webseed(strategy) - }) - - it('Should undo redundancy on server 1 and remove duplicated videos', async function () { - this.timeout(40000) - - await disableRedundancyOnServer1() - - await waitJobs(servers) - await wait(5000) - - await check1WebSeed(strategy) - - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) - }) - - after(function () { - return cleanServers() - }) - }) - - describe('With trending strategy', function () { - const strategy = 'trending' - - before(function () { - this.timeout(120000) - - return runServers(strategy) - }) - - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) - await checkStatsWith1Webseed(strategy) - }) - - it('Should enable redundancy on server 1', function () { - return enableRedundancyOnServer1() - }) - - it('Should have 2 webseed on the first video', async function () { - this.timeout(40000) - - await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) - await waitJobs(servers) - - await check2Webseeds(strategy) - await checkStatsWith2Webseed(strategy) - }) - - it('Should unfollow on server 1 and remove duplicated videos', async function () { - this.timeout(40000) - - await unfollow(servers[0].url, servers[0].accessToken, servers[1]) - - await waitJobs(servers) - await wait(5000) - - await check1WebSeed(strategy) - - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) - }) - - after(function () { - return cleanServers() - }) - }) - - describe('With recently added strategy', function () { - const strategy = 'recently-added' - - before(function () { - this.timeout(120000) - - return runServers(strategy, { min_views: 3 }) - }) - - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) - await checkStatsWith1Webseed(strategy) - }) - - it('Should enable redundancy on server 1', function () { - return enableRedundancyOnServer1() - }) - - it('Should still have 1 webseed on the first video', async function () { - this.timeout(40000) - - await waitJobs(servers) - await wait(15000) - await waitJobs(servers) - - await check1WebSeed(strategy) - await checkStatsWith1Webseed(strategy) - }) - - it('Should view 2 times the first video to have > min_views config', async function () { - this.timeout(40000) - - await viewVideo(servers[ 0 ].url, video1Server2UUID) - await viewVideo(servers[ 2 ].url, video1Server2UUID) - - await wait(10000) - await waitJobs(servers) - }) - - it('Should have 2 webseed on the first video', async function () { - this.timeout(40000) - - await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) - await waitJobs(servers) - - await check2Webseeds(strategy) - await checkStatsWith2Webseed(strategy) - }) - - it('Should remove the video and the redundancy files', async function () { - this.timeout(20000) - - await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID) - - await waitJobs(servers) - - for (const server of servers) { - await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber) - } - }) - - after(function () { - return cleanServers() - }) - }) - - describe('Test expiration', function () { - const strategy = 'recently-added' - - async function checkContains (servers: ServerInfo[], str: string) { - for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) - const video: VideoDetails = res.body - - for (const f of video.files) { - expect(f.magnetUri).to.contain(str) - } - } - } - - async function checkNotContains (servers: ServerInfo[], str: string) { - for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) - const video: VideoDetails = res.body - - for (const f of video.files) { - expect(f.magnetUri).to.not.contain(str) - } - } - } - - before(async function () { - this.timeout(120000) - - await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) - - await enableRedundancyOnServer1() - }) - - it('Should still have 2 webseeds after 10 seconds', async function () { - this.timeout(40000) - - await wait(10000) - - try { - await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') - } catch { - // Maybe a server deleted a redundancy in the scheduler - await wait(2000) - - await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') - } - }) - - it('Should stop server 1 and expire video redundancy', async function () { - this.timeout(40000) - - killallServers([ servers[0] ]) - - await wait(15000) - - await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') - }) - - after(function () { - return killallServers([ servers[1], servers[2] ]) - }) - }) - - describe('Test file replacement', function () { - let video2Server2UUID: string - const strategy = 'recently-added' - - before(async function () { - this.timeout(120000) - - await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) - - await enableRedundancyOnServer1() - - await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) - await waitJobs(servers) - - await check2Webseeds(strategy) - await checkStatsWith2Webseed(strategy) - - const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) - video2Server2UUID = res.body.video.uuid - }) - - it('Should cache video 2 webseed on the first video', async function () { - this.timeout(120000) - - await waitJobs(servers) - - let checked = false - - while (checked === false) { - await wait(1000) - - try { - await check1WebSeed(strategy, video1Server2UUID) - await check2Webseeds(strategy, video2Server2UUID) - - checked = true - } catch { - checked = false - } - } - }) - - after(function () { - return cleanServers() - }) - }) -}) diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index 9858e2b15..aaa6c62f7 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts @@ -75,6 +75,7 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalLocalVideoComments).to.equal(0) expect(data.totalLocalVideos).to.equal(0) expect(data.totalLocalVideoViews).to.equal(0) + expect(data.totalLocalVideoFilesSize).to.equal(0) expect(data.totalUsers).to.equal(1) expect(data.totalVideoComments).to.equal(1) expect(data.totalVideos).to.equal(1) -- cgit v1.2.3 From 1e7eb25f6cb6893db8f99ff40ef0509aa2a16614 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 15 Jan 2019 14:52:33 +0100 Subject: Correctly send Flag/Dislike/View activities --- server/controllers/activitypub/client.ts | 15 +++---- server/controllers/api/videos/abuse.ts | 2 +- server/controllers/api/videos/index.ts | 4 +- server/lib/activitypub/send/send-create.ts | 69 ----------------------------- server/lib/activitypub/send/send-dislike.ts | 41 +++++++++++++++++ server/lib/activitypub/send/send-flag.ts | 39 ++++++++++++++++ server/lib/activitypub/send/send-undo.ts | 12 ++--- server/lib/activitypub/send/send-view.ts | 40 +++++++++++++++++ server/lib/activitypub/video-rates.ts | 5 ++- 9 files changed, 138 insertions(+), 89 deletions(-) create mode 100644 server/lib/activitypub/send/send-dislike.ts create mode 100644 server/lib/activitypub/send/send-flag.ts create mode 100644 server/lib/activitypub/send/send-view.ts (limited to 'server') diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 1a4e28dc8..7e87f6f3b 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -3,22 +3,18 @@ import * as express from 'express' import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' -import { buildAnnounceWithVideoAudience, buildDislikeActivity, buildLikeActivity } from '../../lib/activitypub/send' +import { buildAnnounceWithVideoAudience, buildLikeActivity } from '../../lib/activitypub/send' import { audiencify, getAudience } from '../../lib/activitypub/audience' import { buildCreateActivity } from '../../lib/activitypub/send/send-create' import { asyncMiddleware, - videosShareValidator, executeIfActivityPub, localAccountValidator, localVideoChannelValidator, - videosCustomGetValidator + videosCustomGetValidator, + videosShareValidator } from '../../middlewares' -import { - getAccountVideoRateValidator, - videoCommentGetValidator, - videosGetValidator -} from '../../middlewares/validators' +import { getAccountVideoRateValidator, videoCommentGetValidator, videosGetValidator } from '../../middlewares/validators' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { ActorFollowModel } from '../../models/activitypub/actor-follow' @@ -40,6 +36,7 @@ import { VideoCaptionModel } from '../../models/video/video-caption' import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' import { getServerActor } from '../../helpers/utils' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' +import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike' const activityPubClientRouter = express.Router() @@ -156,7 +153,7 @@ function getAccountVideoRate (rateType: VideoRateType) { const url = getRateUrl(rateType, byActor, accountVideoRate.Video) const APObject = rateType === 'like' ? buildLikeActivity(url, byActor, accountVideoRate.Video) - : buildCreateActivity(url, byActor, buildDislikeActivity(url, byActor, accountVideoRate.Video)) + : buildDislikeActivity(url, byActor, accountVideoRate.Video) return activityPubResponse(activityPubContextify(APObject), res) } diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index fe0a95cd5..32f9c4793 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts @@ -3,7 +3,6 @@ import { UserRight, VideoAbuseCreate, VideoAbuseState } from '../../../../shared import { logger } from '../../../helpers/logger' import { getFormattedObjects } from '../../../helpers/utils' import { sequelizeTypescript } from '../../../initializers' -import { sendVideoAbuse } from '../../../lib/activitypub/send' import { asyncMiddleware, asyncRetryTransactionMiddleware, @@ -23,6 +22,7 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse' import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger' import { UserModel } from '../../../models/account/user' import { Notifier } from '../../../lib/notifier' +import { sendVideoAbuse } from '../../../lib/activitypub/send/send-flag' const auditLogger = auditLoggerFactory('abuse') const abuseVideoRouter = express.Router() diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 2b2dfa7ca..8414ca42c 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -23,7 +23,6 @@ import { fetchRemoteVideoDescription, getVideoActivityPubUrl } from '../../../lib/activitypub' -import { sendCreateView } from '../../../lib/activitypub/send' import { JobQueue } from '../../../lib/job-queue' import { Redis } from '../../../lib/redis' import { @@ -59,6 +58,7 @@ import { resetSequelizeInstance } from '../../../helpers/database-utils' import { move } from 'fs-extra' import { watchingRouter } from './watching' import { Notifier } from '../../../lib/notifier' +import { sendView } from '../../../lib/activitypub/send/send-view' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() @@ -422,7 +422,7 @@ async function viewVideo (req: express.Request, res: express.Response) { ]) const serverActor = await getServerActor() - await sendCreateView(serverActor, videoInstance, undefined) + await sendView(serverActor, videoInstance, undefined) return res.status(204).end() } diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index e3fca0a17..73e667ad4 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -3,9 +3,7 @@ import { ActivityAudience, ActivityCreate } from '../../../../shared/models/acti import { VideoPrivacy } from '../../../../shared/models/videos' import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' -import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' -import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' import { logger } from '../../../helpers/logger' @@ -25,20 +23,6 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { return broadcastToFollowers(createActivity, byActor, [ byActor ], t) } -async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { - if (!video.VideoChannel.Account.Actor.serverId) return // Local - - const url = getVideoAbuseActivityPubUrl(videoAbuse) - - logger.info('Creating job to send video abuse %s.', url) - - // Custom audience, we only send the abuse to the origin instance - const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } - const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) - - return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) -} - async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { logger.info('Creating job to send file cache of %s.', fileRedundancy.url) @@ -91,37 +75,6 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) } -async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { - logger.info('Creating job to send view of %s.', video.url) - - const url = getVideoViewActivityPubUrl(byActor, video) - const viewActivity = buildViewActivity(url, byActor, video) - - return sendVideoRelatedCreateActivity({ - // Use the server actor to send the view - byActor, - video, - url, - object: viewActivity, - transaction: t - }) -} - -async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { - logger.info('Creating job to dislike %s.', video.url) - - const url = getVideoDislikeActivityPubUrl(byActor, video) - const dislikeActivity = buildDislikeActivity(url, byActor, video) - - return sendVideoRelatedCreateActivity({ - byActor, - video, - url, - object: dislikeActivity, - transaction: t - }) -} - function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { if (!audience) audience = getAudience(byActor) @@ -136,33 +89,11 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud ) } -function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel) { - return { - id: url, - type: 'Dislike', - actor: byActor.url, - object: video.url - } -} - -function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel) { - return { - id: url, - type: 'View', - actor: byActor.url, - object: video.url - } -} - // --------------------------------------------------------------------------- export { sendCreateVideo, - sendVideoAbuse, buildCreateActivity, - sendCreateView, - sendCreateDislike, - buildDislikeActivity, sendCreateVideoComment, sendCreateCacheFile } diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts new file mode 100644 index 000000000..a88436f2c --- /dev/null +++ b/server/lib/activitypub/send/send-dislike.ts @@ -0,0 +1,41 @@ +import { Transaction } from 'sequelize' +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoModel } from '../../../models/video/video' +import { getVideoDislikeActivityPubUrl } from '../url' +import { logger } from '../../../helpers/logger' +import { ActivityAudience, ActivityDislike } from '../../../../shared/models/activitypub' +import { sendVideoRelatedActivity } from './utils' +import { audiencify, getAudience } from '../audience' + +async function sendDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { + logger.info('Creating job to dislike %s.', video.url) + + const activityBuilder = (audience: ActivityAudience) => { + const url = getVideoDislikeActivityPubUrl(byActor, video) + + return buildDislikeActivity(url, byActor, video, audience) + } + + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) +} + +function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityDislike { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + id: url, + type: 'Dislike' as 'Dislike', + actor: byActor.url, + object: video.url + }, + audience + ) +} + +// --------------------------------------------------------------------------- + +export { + sendDislike, + buildDislikeActivity +} diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts new file mode 100644 index 000000000..96a7311b9 --- /dev/null +++ b/server/lib/activitypub/send/send-flag.ts @@ -0,0 +1,39 @@ +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoModel } from '../../../models/video/video' +import { VideoAbuseModel } from '../../../models/video/video-abuse' +import { getVideoAbuseActivityPubUrl } from '../url' +import { unicastTo } from './utils' +import { logger } from '../../../helpers/logger' +import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub' +import { audiencify, getAudience } from '../audience' + +async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { + if (!video.VideoChannel.Account.Actor.serverId) return // Local user + + const url = getVideoAbuseActivityPubUrl(videoAbuse) + + logger.info('Creating job to send video abuse %s.', url) + + // Custom audience, we only send the abuse to the origin instance + const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } + const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience) + + return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) +} + +function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag { + if (!audience) audience = getAudience(byActor) + + const activity = Object.assign( + { id: url, actor: byActor.url }, + videoAbuse.toActivityPubObject() + ) + + return audiencify(activity, audience) +} + +// --------------------------------------------------------------------------- + +export { + sendVideoAbuse +} diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index bf1b6e117..eb18a6cb6 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -2,7 +2,7 @@ import { Transaction } from 'sequelize' import { ActivityAnnounce, ActivityAudience, - ActivityCreate, + ActivityCreate, ActivityDislike, ActivityFollow, ActivityLike, ActivityUndo @@ -13,13 +13,14 @@ import { VideoModel } from '../../../models/video/video' import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' import { audiencify, getAudience } from '../audience' -import { buildCreateActivity, buildDislikeActivity } from './send-create' +import { buildCreateActivity } from './send-create' import { buildFollowActivity } from './send-follow' import { buildLikeActivity } from './send-like' import { VideoShareModel } from '../../../models/video/video-share' import { buildAnnounceWithVideoAudience } from './send-announce' import { logger } from '../../../helpers/logger' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' +import { buildDislikeActivity } from './send-dislike' async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { const me = actorFollow.ActorFollower @@ -65,9 +66,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) - const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) - return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) + return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) } async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { @@ -94,7 +94,7 @@ export { function undoActivityData ( url: string, byActor: ActorModel, - object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, + object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, audience?: ActivityAudience ): ActivityUndo { if (!audience) audience = getAudience(byActor) @@ -114,7 +114,7 @@ async function sendUndoVideoRelatedActivity (options: { byActor: ActorModel, video: VideoModel, url: string, - activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, + activity: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, transaction: Transaction }) { const activityBuilder = (audience: ActivityAudience) => { diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts new file mode 100644 index 000000000..8ad126be0 --- /dev/null +++ b/server/lib/activitypub/send/send-view.ts @@ -0,0 +1,40 @@ +import { Transaction } from 'sequelize' +import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' +import { ActorModel } from '../../../models/activitypub/actor' +import { VideoModel } from '../../../models/video/video' +import { getVideoLikeActivityPubUrl } from '../url' +import { sendVideoRelatedActivity } from './utils' +import { audiencify, getAudience } from '../audience' +import { logger } from '../../../helpers/logger' + +async function sendView (byActor: ActorModel, video: VideoModel, t: Transaction) { + logger.info('Creating job to send view of %s.', video.url) + + const activityBuilder = (audience: ActivityAudience) => { + const url = getVideoLikeActivityPubUrl(byActor, video) + + return buildViewActivity(url, byActor, video, audience) + } + + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) +} + +function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityView { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + id: url, + type: 'View' as 'View', + actor: byActor.url, + object: video.url + }, + audience + ) +} + +// --------------------------------------------------------------------------- + +export { + sendView +} diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 45a2b22ea..7aac79118 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -1,7 +1,7 @@ import { Transaction } from 'sequelize' import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' -import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' +import { sendLike, sendUndoDislike, sendUndoLike } from './send' import { VideoRateType } from '../../../shared/models/videos' import * as Bluebird from 'bluebird' import { getOrCreateActorAndServerAndModel } from './actor' @@ -12,6 +12,7 @@ import { doRequest } from '../../helpers/requests' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { ActorModel } from '../../models/activitypub/actor' import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' +import { sendDislike } from './send/send-dislike' async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { let rateCounts = 0 @@ -82,7 +83,7 @@ async function sendVideoRateChange (account: AccountModel, // Like if (likes > 0) await sendLike(actor, video, t) // Dislike - if (dislikes > 0) await sendCreateDislike(actor, video, t) + if (dislikes > 0) await sendDislike(actor, video, t) } function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { -- cgit v1.2.3 From 457bb213b273a9b206cc5654eb085cede4e916ad Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 16 Jan 2019 16:05:40 +0100 Subject: Refactor how we use icons Inject them in an angular component so we can easily change their color --- server/models/account/user-notification.ts | 158 ++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 38 deletions(-) (limited to 'server') diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts index 9e4f982a3..1094eec78 100644 --- a/server/models/account/user-notification.ts +++ b/server/models/account/user-notification.ts @@ -27,11 +27,27 @@ import { VideoBlacklistModel } from '../video/video-blacklist' import { VideoImportModel } from '../video/video-import' import { ActorModel } from '../activitypub/actor' import { ActorFollowModel } from '../activitypub/actor-follow' +import { AvatarModel } from '../avatar/avatar' enum ScopeNames { WITH_ALL = 'WITH_ALL' } +function buildActorWithAvatarInclude () { + return { + attributes: [ 'preferredUsername' ], + model: () => ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'filename' ], + model: () => AvatarModel.unscoped(), + required: false + } + ] + } +} + function buildVideoInclude (required: boolean) { return { attributes: [ 'id', 'uuid', 'name' ], @@ -40,19 +56,21 @@ function buildVideoInclude (required: boolean) { } } -function buildChannelInclude (required: boolean) { +function buildChannelInclude (required: boolean, withActor = false) { return { required, attributes: [ 'id', 'name' ], - model: () => VideoChannelModel.unscoped() + model: () => VideoChannelModel.unscoped(), + include: withActor === true ? [ buildActorWithAvatarInclude() ] : [] } } -function buildAccountInclude (required: boolean) { +function buildAccountInclude (required: boolean, withActor = false) { return { required, attributes: [ 'id', 'name' ], - model: () => AccountModel.unscoped() + model: () => AccountModel.unscoped(), + include: withActor === true ? [ buildActorWithAvatarInclude() ] : [] } } @@ -60,47 +78,40 @@ function buildAccountInclude (required: boolean) { [ScopeNames.WITH_ALL]: { include: [ Object.assign(buildVideoInclude(false), { - include: [ buildChannelInclude(true) ] + include: [ buildChannelInclude(true, true) ] }), + { attributes: [ 'id', 'originCommentId' ], model: () => VideoCommentModel.unscoped(), required: false, include: [ - buildAccountInclude(true), + buildAccountInclude(true, true), buildVideoInclude(true) ] }, + { attributes: [ 'id' ], model: () => VideoAbuseModel.unscoped(), required: false, include: [ buildVideoInclude(true) ] }, + { attributes: [ 'id' ], model: () => VideoBlacklistModel.unscoped(), required: false, include: [ buildVideoInclude(true) ] }, + { attributes: [ 'id', 'magnetUri', 'targetUrl', 'torrentName' ], model: () => VideoImportModel.unscoped(), required: false, include: [ buildVideoInclude(false) ] }, - { - attributes: [ 'id', 'name' ], - model: () => AccountModel.unscoped(), - required: false, - include: [ - { - attributes: [ 'id', 'preferredUsername' ], - model: () => ActorModel.unscoped(), - required: true - } - ] - }, + { attributes: [ 'id' ], model: () => ActorFollowModel.unscoped(), @@ -111,7 +122,18 @@ function buildAccountInclude (required: boolean) { model: () => ActorModel.unscoped(), required: true, as: 'ActorFollower', - include: [ buildAccountInclude(true) ] + include: [ + { + attributes: [ 'id', 'name' ], + model: () => AccountModel.unscoped(), + required: true + }, + { + attributes: [ 'filename' ], + model: () => AvatarModel.unscoped(), + required: false + } + ] }, { attributes: [ 'preferredUsername' ], @@ -124,7 +146,9 @@ function buildAccountInclude (required: boolean) { ] } ] - } + }, + + buildAccountInclude(false, true) ] } }) @@ -132,10 +156,63 @@ function buildAccountInclude (required: boolean) { tableName: 'userNotification', indexes: [ { - fields: [ 'videoId' ] + fields: [ 'userId' ] }, { - fields: [ 'commentId' ] + fields: [ 'videoId' ], + where: { + videoId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'commentId' ], + where: { + commentId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoAbuseId' ], + where: { + videoAbuseId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoBlacklistId' ], + where: { + videoBlacklistId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoImportId' ], + where: { + videoImportId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'accountId' ], + where: { + accountId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'actorFollowId' ], + where: { + actorFollowId: { + [Op.ne]: null + } + } } ] }) @@ -297,12 +374,9 @@ export class UserNotificationModel extends Model { } toFormattedJSON (): UserNotification { - const video = this.Video ? Object.assign(this.formatVideo(this.Video), { - channel: { - id: this.Video.VideoChannel.id, - displayName: this.Video.VideoChannel.getDisplayName() - } - }) : undefined + const video = this.Video + ? Object.assign(this.formatVideo(this.Video),{ channel: this.formatActor(this.Video.VideoChannel) }) + : undefined const videoImport = this.VideoImport ? { id: this.VideoImport.id, @@ -315,10 +389,7 @@ export class UserNotificationModel extends Model { const comment = this.Comment ? { id: this.Comment.id, threadId: this.Comment.getThreadId(), - account: { - id: this.Comment.Account.id, - displayName: this.Comment.Account.getDisplayName() - }, + account: this.formatActor(this.Comment.Account), video: this.formatVideo(this.Comment.Video) } : undefined @@ -332,17 +403,15 @@ export class UserNotificationModel extends Model { video: this.formatVideo(this.VideoBlacklist.Video) } : undefined - const account = this.Account ? { - id: this.Account.id, - displayName: this.Account.getDisplayName(), - name: this.Account.Actor.preferredUsername - } : undefined + const account = this.Account ? this.formatActor(this.Account) : undefined const actorFollow = this.ActorFollow ? { id: this.ActorFollow.id, follower: { + id: this.ActorFollow.ActorFollower.Account.id, displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), - name: this.ActorFollow.ActorFollower.preferredUsername + name: this.ActorFollow.ActorFollower.preferredUsername, + avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined }, following: { type: this.ActorFollow.ActorFollowing.VideoChannel ? 'channel' as 'channel' : 'account' as 'account', @@ -374,4 +443,17 @@ export class UserNotificationModel extends Model { name: video.name } } + + private formatActor (accountOrChannel: AccountModel | VideoChannelModel) { + const avatar = accountOrChannel.Actor.Avatar + ? { path: accountOrChannel.Actor.Avatar.getWebserverPath() } + : undefined + + return { + id: accountOrChannel.id, + displayName: accountOrChannel.getDisplayName(), + name: accountOrChannel.Actor.preferredUsername, + avatar + } + } } -- cgit v1.2.3 From ef04ae20fe4155f516ab41959e312de093f98d0e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 17 Jan 2019 14:03:32 +0100 Subject: Prefer avg_frame_rate to fetch video FPS --- server/helpers/ffmpeg-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index c7296054d..132f4690e 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -41,7 +41,7 @@ async function getVideoFileResolution (path: string) { async function getVideoFileFPS (path: string) { const videoStream = await getVideoFileStream(path) - for (const key of [ 'r_frame_rate' , 'avg_frame_rate' ]) { + for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { const valuesText: string = videoStream[key] if (!valuesText) continue -- cgit v1.2.3 From 38967f7b73cec6f6198c72d62f8d64bb88e6951c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 21 Jan 2019 13:52:46 +0100 Subject: Add server host in notification account field --- server/models/account/user-notification.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'server') diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts index 1094eec78..6cdbb827b 100644 --- a/server/models/account/user-notification.ts +++ b/server/models/account/user-notification.ts @@ -28,6 +28,7 @@ import { VideoImportModel } from '../video/video-import' import { ActorModel } from '../activitypub/actor' import { ActorFollowModel } from '../activitypub/actor-follow' import { AvatarModel } from '../avatar/avatar' +import { ServerModel } from '../server/server' enum ScopeNames { WITH_ALL = 'WITH_ALL' @@ -43,6 +44,11 @@ function buildActorWithAvatarInclude () { attributes: [ 'filename' ], model: () => AvatarModel.unscoped(), required: false + }, + { + attributes: [ 'host' ], + model: () => ServerModel.unscoped(), + required: false } ] } @@ -132,6 +138,11 @@ function buildAccountInclude (required: boolean, withActor = false) { attributes: [ 'filename' ], model: () => AvatarModel.unscoped(), required: false + }, + { + attributes: [ 'host' ], + model: () => ServerModel.unscoped(), + required: false } ] }, @@ -411,7 +422,8 @@ export class UserNotificationModel extends Model { id: this.ActorFollow.ActorFollower.Account.id, displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), name: this.ActorFollow.ActorFollower.preferredUsername, - avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined + avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined, + host: this.ActorFollow.ActorFollower.getHost() }, following: { type: this.ActorFollow.ActorFollowing.VideoChannel ? 'channel' as 'channel' : 'account' as 'account', @@ -453,6 +465,7 @@ export class UserNotificationModel extends Model { id: accountOrChannel.id, displayName: accountOrChannel.getDisplayName(), name: accountOrChannel.Actor.preferredUsername, + host: accountOrChannel.Actor.getHost(), avatar } } -- cgit v1.2.3 From ebff55d8d6747d0627f135ab668f43e0d6125a37 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 21 Jan 2019 15:58:07 +0100 Subject: Fix tests --- server/initializers/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 93fdd3f03..dab986353 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -149,7 +149,7 @@ let SCHEDULER_INTERVALS_MS = { actorFollowScores: 60000 * 60, // 1 hour removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute - youtubeDLUpdate: 60000 * 60 * 24 // 1 day + youtubeDLUpdate: 60000 // 1 day } // --------------------------------------------------------------------------- -- cgit v1.2.3 From 0c5892764e74f4a614f0087028e0687694b176d3 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 21 Jan 2019 16:22:15 +0100 Subject: Youtube DL update every 24 hours --- server/initializers/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index dab986353..93fdd3f03 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -149,7 +149,7 @@ let SCHEDULER_INTERVALS_MS = { actorFollowScores: 60000 * 60, // 1 hour removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute - youtubeDLUpdate: 60000 // 1 day + youtubeDLUpdate: 60000 * 60 * 24 // 1 day } // --------------------------------------------------------------------------- -- cgit v1.2.3 From 307902e2b3248073aeb677e420aafd8b5e041117 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 24 Jan 2019 15:23:06 +0100 Subject: Try to fix Mac video upload --- server/helpers/custom-validators/videos.ts | 4 ++-- server/initializers/constants.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'server') diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index e6f22e6c5..95e256b8f 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -88,8 +88,8 @@ function isVideoFileExtnameValid (value: string) { function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { const videoFileTypesRegex = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) - .map(m => `(${m})`) - .join('|') + .map(m => `(${m})`) + .join('|') return isFileValid(files, videoFileTypesRegex, 'videofile', null) } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 93fdd3f03..6f3ebb9aa 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -795,7 +795,9 @@ function buildVideoMimetypeExt () { 'video/quicktime': '.mov', 'video/x-msvideo': '.avi', 'video/x-flv': '.flv', - 'video/x-matroska': '.mkv' + 'video/x-matroska': '.mkv', + 'application/octet-stream': '.mkv', + 'video/avi': '.avi' }) } -- cgit v1.2.3 From 926cd3df339772dd1cbb9e10996518e8cb2e001d Mon Sep 17 00:00:00 2001 From: Josh Morel Date: Mon, 28 Jan 2019 05:45:40 -0500 Subject: fix typo in test accounts api validators --- server/tests/api/check-params/accounts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/tests/api/check-params/accounts.ts b/server/tests/api/check-params/accounts.ts index 567fd072c..68f9519c6 100644 --- a/server/tests/api/check-params/accounts.ts +++ b/server/tests/api/check-params/accounts.ts @@ -10,7 +10,7 @@ import { } from '../../../../shared/utils/requests/check-api-params' import { getAccount } from '../../../../shared/utils/users/accounts' -describe('Test users API validators', function () { +describe('Test accounts API validators', function () { const path = '/api/v1/accounts/' let server: ServerInfo -- cgit v1.2.3 From f7effe8dc7c641388f7edbcaad716fc16321d794 Mon Sep 17 00:00:00 2001 From: Josh Morel Date: Wed, 6 Feb 2019 06:14:45 -0500 Subject: don't notify prior to scheduled update also increase timeouts on user-notification test --- server/lib/job-queue/handlers/video-file.ts | 10 +++++--- server/tests/api/users/user-notifications.ts | 38 +++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) (limited to 'server') diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 593e43cc5..217d666b6 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -91,7 +91,8 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { return { videoDatabase, videoPublished } }) - if (videoPublished) { + // don't notify prior to scheduled video update + if (videoPublished && !videoDatabase.ScheduleVideoUpdate) { Notifier.Instance.notifyOnNewVideo(videoDatabase) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } @@ -149,8 +150,11 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo return { videoDatabase, videoPublished } }) - if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) - if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) + // don't notify prior to scheduled video update + if (!videoDatabase.ScheduleVideoUpdate) { + if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) + } } // --------------------------------------------------------------------------- diff --git a/server/tests/api/users/user-notifications.ts b/server/tests/api/users/user-notifications.ts index 5260d64cc..317f68176 100644 --- a/server/tests/api/users/user-notifications.ts +++ b/server/tests/api/users/user-notifications.ts @@ -79,7 +79,7 @@ async function uploadVideoByLocalAccount (servers: ServerInfo[], additionalParam return { uuid: res.body.video.uuid, name } } -describe('Test users notifications', function () { +describe.only('Test users notifications', function () { let servers: ServerInfo[] = [] let userAccessToken: string let userNotifications: UserNotification[] = [] @@ -165,6 +165,8 @@ describe('Test users notifications', function () { }) it('Should not send notifications if the user does not follow the video publisher', async function () { + this.timeout(10000) + await uploadVideoByLocalAccount(servers) const notification = await getLastNotification(servers[ 0 ].url, userAccessToken) @@ -644,6 +646,8 @@ describe('Test users notifications', function () { }) it('Should not send a notification if transcoding is not enabled', async function () { + this.timeout(10000) + const { name, uuid } = await uploadVideoByLocalAccount(servers) await waitJobs(servers) @@ -717,6 +721,24 @@ describe('Test users notifications', function () { await wait(6000) await checkVideoIsPublished(baseParams, name, uuid, 'presence') }) + + it('Should not send a notification before the video is published', async function () { + this.timeout(20000) + + let updateAt = new Date(new Date().getTime() + 100000) + + const data = { + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + const { name, uuid } = await uploadVideoByRemoteAccount(servers, data) + + await wait(6000) + await checkVideoIsPublished(baseParams, name, uuid, 'absence') + }) }) describe('My video is imported', function () { @@ -781,6 +803,8 @@ describe('Test users notifications', function () { }) it('Should send a notification only to moderators when a user registers on the instance', async function () { + this.timeout(10000) + await registerUser(servers[0].url, 'user_45', 'password') await waitJobs(servers) @@ -849,6 +873,8 @@ describe('Test users notifications', function () { }) it('Should notify when a local account is following one of our channel', async function () { + this.timeout(10000) + await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1@localhost:9001') await waitJobs(servers) @@ -857,6 +883,8 @@ describe('Test users notifications', function () { }) it('Should notify when a remote account is following one of our channel', async function () { + this.timeout(10000) + await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1@localhost:9001') await waitJobs(servers) @@ -926,6 +954,8 @@ describe('Test users notifications', function () { }) it('Should not have notifications', async function () { + this.timeout(10000) + await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.NONE })) @@ -943,6 +973,8 @@ describe('Test users notifications', function () { }) it('Should only have web notifications', async function () { + this.timeout(10000) + await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.WEB })) @@ -967,6 +999,8 @@ describe('Test users notifications', function () { }) it('Should only have mail notifications', async function () { + this.timeout(10000) + await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.EMAIL })) @@ -991,6 +1025,8 @@ describe('Test users notifications', function () { }) it('Should have email and web notifications', async function () { + this.timeout(10000) + await updateMyNotificationSettings(servers[0].url, userAccessToken, immutableAssign(allNotificationSettings, { newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL })) -- cgit v1.2.3 From 6c32d302126f455779f5593192775b86833f1f33 Mon Sep 17 00:00:00 2001 From: Josh Morel Date: Thu, 7 Feb 2019 05:48:17 -0500 Subject: remove .only from notifications tests --- server/tests/api/users/user-notifications.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/tests/api/users/user-notifications.ts b/server/tests/api/users/user-notifications.ts index 317f68176..69e51677e 100644 --- a/server/tests/api/users/user-notifications.ts +++ b/server/tests/api/users/user-notifications.ts @@ -79,7 +79,7 @@ async function uploadVideoByLocalAccount (servers: ServerInfo[], additionalParam return { uuid: res.body.video.uuid, name } } -describe.only('Test users notifications', function () { +describe('Test users notifications', function () { let servers: ServerInfo[] = [] let userAccessToken: string let userNotifications: UserNotification[] = [] -- cgit v1.2.3 From 2adfc7ea9a1f858db874df9fe322e7ae833db77c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 23 Jan 2019 15:36:45 +0100 Subject: Refractor videojs player Add fake p2p-media-loader plugin --- server/middlewares/csp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/middlewares/csp.ts b/server/middlewares/csp.ts index 8b919af0d..5fa9d1ab5 100644 --- a/server/middlewares/csp.ts +++ b/server/middlewares/csp.ts @@ -16,7 +16,7 @@ const baseDirectives = Object.assign({}, baseUri: ["'self'"], manifestSrc: ["'self'"], frameSrc: ["'self'"], // instead of deprecated child-src / self because of test-embed - workerSrc: ["'self'"] // instead of deprecated child-src + workerSrc: ["'self'", 'blob:'] // instead of deprecated child-src }, CONFIG.SERVICES['CSP-LOGGER'] ? { reportUri: CONFIG.SERVICES['CSP-LOGGER'] } : {}, CONFIG.WEBSERVER.SCHEME === 'https' ? { upgradeInsecureRequests: true } : {} -- cgit v1.2.3 From 092092969633bbcf6d4891a083ea497a7d5c3154 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 29 Jan 2019 08:37:25 +0100 Subject: Add hls support on server --- server/controllers/activitypub/client.ts | 15 +- server/controllers/api/config.ts | 8 +- server/controllers/api/videos/index.ts | 19 +- server/controllers/static.ts | 9 +- server/controllers/tracker.ts | 25 ++- server/helpers/activitypub.ts | 5 +- server/helpers/core-utils.ts | 8 +- .../custom-validators/activitypub/cache-file.ts | 12 +- .../custom-validators/activitypub/videos.ts | 8 +- server/helpers/custom-validators/misc.ts | 5 + server/helpers/ffmpeg-utils.ts | 32 +++- server/helpers/video.ts | 4 +- server/initializers/checker-before-init.ts | 2 +- server/initializers/constants.ts | 14 +- server/initializers/database.ts | 4 +- server/initializers/installer.ts | 5 +- .../migrations/0330-video-streaming-playlist.ts | 51 +++++ server/lib/activitypub/cache-file.ts | 23 ++- server/lib/activitypub/send/send-create.ts | 9 +- server/lib/activitypub/send/send-undo.ts | 3 +- server/lib/activitypub/send/send-update.ts | 2 +- server/lib/activitypub/url.ts | 7 + server/lib/activitypub/videos.ts | 97 +++++++++- server/lib/hls.ts | 110 +++++++++++ server/lib/job-queue/handlers/video-file.ts | 59 +++++- .../lib/schedulers/videos-redundancy-scheduler.ts | 189 ++++++++++++------ server/lib/video-transcoding.ts | 49 ++++- server/middlewares/validators/redundancy.ts | 33 +++- server/models/redundancy/video-redundancy.ts | 139 ++++++++++---- server/models/video/video-file.ts | 6 +- server/models/video/video-format-utils.ts | 61 +++++- server/models/video/video-streaming-playlist.ts | 154 +++++++++++++++ server/models/video/video.ts | 179 ++++++++++++++--- server/tests/api/check-params/config.ts | 3 + server/tests/api/redundancy/redundancy.ts | 212 ++++++++++++++------- server/tests/api/server/config.ts | 6 + server/tests/api/videos/index.ts | 1 + server/tests/api/videos/video-hls.ts | 145 ++++++++++++++ server/tests/cli/update-host.ts | 11 +- 39 files changed, 1452 insertions(+), 272 deletions(-) create mode 100644 server/initializers/migrations/0330-video-streaming-playlist.ts create mode 100644 server/lib/hls.ts create mode 100644 server/models/video/video-streaming-playlist.ts create mode 100644 server/tests/api/videos/video-hls.ts (limited to 'server') diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 1a4e28dc8..32a83aa5f 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -37,7 +37,7 @@ import { getVideoSharesActivityPubUrl } from '../../lib/activitypub' import { VideoCaptionModel } from '../../models/video/video-caption' -import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' +import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' import { getServerActor } from '../../helpers/utils' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' @@ -66,11 +66,11 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId', activityPubClientRouter.get('/videos/watch/:id', executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), - executeIfActivityPub(asyncMiddleware(videosGetValidator)), + executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))), executeIfActivityPub(asyncMiddleware(videoController)) ) activityPubClientRouter.get('/videos/watch/:id/activity', - executeIfActivityPub(asyncMiddleware(videosGetValidator)), + executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))), executeIfActivityPub(asyncMiddleware(videoController)) ) activityPubClientRouter.get('/videos/watch/:id/announces', @@ -116,7 +116,11 @@ activityPubClientRouter.get('/video-channels/:name/following', ) activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', - executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), + executeIfActivityPub(asyncMiddleware(videoFileRedundancyGetValidator)), + executeIfActivityPub(asyncMiddleware(videoRedundancyController)) +) +activityPubClientRouter.get('/redundancy/video-playlists/:streamingPlaylistType/:videoId', + executeIfActivityPub(asyncMiddleware(videoPlaylistRedundancyGetValidator)), executeIfActivityPub(asyncMiddleware(videoRedundancyController)) ) @@ -163,7 +167,8 @@ function getAccountVideoRate (rateType: VideoRateType) { } async function videoController (req: express.Request, res: express.Response) { - const video: VideoModel = res.locals.video + // We need more attributes + const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id) if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 255026f46..1f3341bc0 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -1,5 +1,5 @@ import * as express from 'express' -import { omit, snakeCase } from 'lodash' +import { snakeCase } from 'lodash' import { ServerConfig, UserRight } from '../../../shared' import { About } from '../../../shared/models/server/about.model' import { CustomConfig } from '../../../shared/models/server/custom-config.model' @@ -78,6 +78,9 @@ async function getConfig (req: express.Request, res: express.Response) { requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION }, transcoding: { + hls: { + enabled: CONFIG.TRANSCODING.HLS.ENABLED + }, enabledResolutions }, import: { @@ -246,6 +249,9 @@ function customConfig (): CustomConfig { '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] + }, + hls: { + enabled: CONFIG.TRANSCODING.HLS.ENABLED } }, import: { diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 2b2dfa7ca..e04fc8186 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -37,6 +37,7 @@ import { setDefaultPagination, setDefaultSort, videosAddValidator, + videosCustomGetValidator, videosGetValidator, videosRemoveValidator, videosSortValidator, @@ -123,9 +124,9 @@ videosRouter.get('/:id/description', ) videosRouter.get('/:id', optionalAuthenticate, - asyncMiddleware(videosGetValidator), + asyncMiddleware(videosCustomGetValidator('only-video-with-rights')), asyncMiddleware(checkVideoFollowConstraints), - getVideo + asyncMiddleware(getVideo) ) videosRouter.post('/:id/views', asyncMiddleware(videosGetValidator), @@ -395,15 +396,17 @@ async function updateVideo (req: express.Request, res: express.Response) { return res.type('json').status(204).end() } -function getVideo (req: express.Request, res: express.Response) { - const videoInstance = res.locals.video +async function getVideo (req: express.Request, res: express.Response) { + // We need more attributes + const userId: number = res.locals.oauth ? res.locals.oauth.token.User.id : null + const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id, undefined, userId) - if (videoInstance.isOutdated()) { - JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) - .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) + if (video.isOutdated()) { + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } }) + .catch(err => logger.error('Cannot create AP refresher job for video %s.', video.url, { err })) } - return res.json(videoInstance.toFormattedDetailsJSON()) + return res.json(video.toFormattedDetailsJSON()) } async function viewVideo (req: express.Request, res: express.Response) { diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 4fd58f70c..b21f9da00 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -1,6 +1,6 @@ import * as cors from 'cors' import * as express from 'express' -import { CONFIG, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' +import { CONFIG, HLS_PLAYLIST_DIRECTORY, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' import { VideosPreviewCache } from '../lib/cache' import { cacheRoute } from '../middlewares/cache' import { asyncMiddleware, videosGetValidator } from '../middlewares' @@ -51,6 +51,13 @@ staticRouter.use( asyncMiddleware(downloadVideoFile) ) +// HLS +staticRouter.use( + STATIC_PATHS.PLAYLISTS.HLS, + cors(), + express.static(HLS_PLAYLIST_DIRECTORY, { fallthrough: false }) // 404 if the file does not exist +) + // Thumbnails path for express const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR staticRouter.use( diff --git a/server/controllers/tracker.ts b/server/controllers/tracker.ts index 1deb8c402..8b77d9de7 100644 --- a/server/controllers/tracker.ts +++ b/server/controllers/tracker.ts @@ -7,6 +7,7 @@ import { Server as WebSocketServer } from 'ws' import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' import { VideoFileModel } from '../models/video/video-file' import { parse } from 'url' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' const TrackerServer = bitTorrentTracker.Server @@ -21,7 +22,7 @@ const trackerServer = new TrackerServer({ udp: false, ws: false, dht: false, - filter: function (infoHash, params, cb) { + filter: async function (infoHash, params, cb) { let ip: string if (params.type === 'ws') { @@ -32,19 +33,25 @@ const trackerServer = new TrackerServer({ const key = ip + '-' + infoHash - peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 - peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 + peersIps[ ip ] = peersIps[ ip ] ? peersIps[ ip ] + 1 : 1 + peersIpInfoHash[ key ] = peersIpInfoHash[ key ] ? peersIpInfoHash[ key ] + 1 : 1 - if (peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { + if (peersIpInfoHash[ key ] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) } - VideoFileModel.isInfohashExists(infoHash) - .then(exists => { - if (exists === false) return cb(new Error(`Unknown infoHash ${infoHash}`)) + try { + const videoFileExists = await VideoFileModel.doesInfohashExist(infoHash) + if (videoFileExists === true) return cb() - return cb() - }) + const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExist(infoHash) + if (playlistExists === true) return cb() + + return cb(new Error(`Unknown infoHash ${infoHash}`)) + } catch (err) { + logger.error('Error in tracker filter.', { err }) + return cb(err) + } } }) diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index f1430055f..eba552524 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -15,7 +15,7 @@ function activityPubContextify (data: T) { 'https://w3id.org/security/v1', { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', - pt: 'https://joinpeertube.org/ns', + pt: 'https://joinpeertube.org/ns#', sc: 'http://schema.org#', Hashtag: 'as:Hashtag', uuid: 'sc:identifier', @@ -32,7 +32,8 @@ function activityPubContextify (data: T) { waitTranscoding: 'sc:Boolean', expires: 'sc:expires', support: 'sc:Text', - CacheFile: 'pt:CacheFile' + CacheFile: 'pt:CacheFile', + Infohash: 'pt:Infohash' }, { likes: { diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 3fb824e36..f38b82d97 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -193,10 +193,14 @@ function peertubeTruncate (str: string, maxLength: number) { return truncate(str, options) } -function sha256 (str: string, encoding: HexBase64Latin1Encoding = 'hex') { +function sha256 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') { return createHash('sha256').update(str).digest(encoding) } +function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') { + return createHash('sha1').update(str).digest(encoding) +} + function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { return function promisified (): Promise { return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { @@ -262,7 +266,9 @@ export { sanitizeHost, buildPath, peertubeTruncate, + sha256, + sha1, promisify0, promisify1, diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts index e2bd0c55e..21d5c53ca 100644 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ b/server/helpers/custom-validators/activitypub/cache-file.ts @@ -8,9 +8,19 @@ function isCacheFileObjectValid (object: CacheFileObject) { object.type === 'CacheFile' && isDateValid(object.expires) && isActivityPubUrlValid(object.object) && - isRemoteVideoUrlValid(object.url) + (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) } +// --------------------------------------------------------------------------- + export { isCacheFileObjectValid } + +// --------------------------------------------------------------------------- + +function isPlaylistRedundancyUrlValid (url: any) { + return url.type === 'Link' && + (url.mediaType || url.mimeType) === 'application/x-mpegURL' && + isActivityPubUrlValid(url.href) +} diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 0f34aab21..ad99c2724 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -1,7 +1,7 @@ import * as validator from 'validator' import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' import { peertubeTruncate } from '../../core-utils' -import { exists, isBooleanValid, isDateValid, isUUIDValid } from '../misc' +import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' import { isVideoDurationValid, isVideoNameValid, @@ -12,7 +12,6 @@ import { } from '../videos' import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' import { VideoState } from '../../../../shared/models/videos' -import { isVideoAbuseReasonValid } from '../video-abuses' function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { return isBaseActivityValid(activity, 'Update') && @@ -81,6 +80,11 @@ function isRemoteVideoUrlValid (url: any) { ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && validator.isLength(url.href, { min: 5 }) && validator.isInt(url.height + '', { min: 0 }) + ) || + ( + (url.mediaType || url.mimeType) === 'application/x-mpegURL' && + isActivityPubUrlValid(url.href) && + isArray(url.tag) ) } diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index b6f0ebe6f..76647fea2 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -13,6 +13,10 @@ function isNotEmptyIntArray (value: any) { return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 } +function isArrayOf (value: any, validator: (value: any) => boolean) { + return isArray(value) && value.every(v => validator(v)) +} + function isDateValid (value: string) { return exists(value) && validator.isISO8601(value) } @@ -82,6 +86,7 @@ function isFileValid ( export { exists, + isArrayOf, isNotEmptyIntArray, isArray, isIdValid, diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 132f4690e..5ad8ed48e 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -1,5 +1,5 @@ import * as ffmpeg from 'fluent-ffmpeg' -import { join } from 'path' +import { dirname, join } from 'path' import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' import { processImage } from './image-utils' @@ -29,12 +29,21 @@ function computeResolutionsToTranscode (videoFileHeight: number) { return resolutionsEnabled } -async function getVideoFileResolution (path: string) { +async function getVideoFileSize (path: string) { const videoStream = await getVideoFileStream(path) return { - videoFileResolution: Math.min(videoStream.height, videoStream.width), - isPortraitMode: videoStream.height > videoStream.width + width: videoStream.width, + height: videoStream.height + } +} + +async function getVideoFileResolution (path: string) { + const size = await getVideoFileSize(path) + + return { + videoFileResolution: Math.min(size.height, size.width), + isPortraitMode: size.height > size.width } } @@ -110,8 +119,10 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima type TranscodeOptions = { inputPath: string outputPath: string - resolution?: VideoResolution + resolution: VideoResolution isPortraitMode?: boolean + + generateHlsPlaylist?: boolean } function transcode (options: TranscodeOptions) { @@ -150,6 +161,16 @@ function transcode (options: TranscodeOptions) { command = command.withFPS(fps) } + if (options.generateHlsPlaylist) { + const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts` + + command = command.outputOption('-hls_time 4') + .outputOption('-hls_list_size 0') + .outputOption('-hls_playlist_type vod') + .outputOption('-hls_segment_filename ' + segmentFilename) + .outputOption('-f hls') + } + command .on('error', (err, stdout, stderr) => { logger.error('Error in transcoding job.', { stdout, stderr }) @@ -166,6 +187,7 @@ function transcode (options: TranscodeOptions) { // --------------------------------------------------------------------------- export { + getVideoFileSize, getVideoFileResolution, getDurationFromVideoFile, generateImageFromVideoFile, diff --git a/server/helpers/video.ts b/server/helpers/video.ts index 1bd21467d..c90fe06c7 100644 --- a/server/helpers/video.ts +++ b/server/helpers/video.ts @@ -1,10 +1,12 @@ import { VideoModel } from '../models/video/video' -type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' +type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) + if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id) + if (fetchType === 'only-video') return VideoModel.load(id) if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 7905d9ffa..29fdb263e 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -12,7 +12,7 @@ function checkMissedConfig () { 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', - 'storage.redundancy', 'storage.tmp', + 'storage.redundancy', 'storage.tmp', 'storage.playlists', 'log.level', 'user.video_quota', 'user.video_quota_daily', 'cache.previews.size', 'admin.email', 'contact_form.enabled', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6f3ebb9aa..98f8f8694 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -16,7 +16,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 325 +const LAST_MIGRATION_VERSION = 330 // --------------------------------------------------------------------------- @@ -192,6 +192,7 @@ const CONFIG = { AVATARS_DIR: buildPath(config.get('storage.avatars')), LOG_DIR: buildPath(config.get('storage.logs')), VIDEOS_DIR: buildPath(config.get('storage.videos')), + PLAYLISTS_DIR: buildPath(config.get('storage.playlists')), REDUNDANCY_DIR: buildPath(config.get('storage.redundancy')), THUMBNAILS_DIR: buildPath(config.get('storage.thumbnails')), PREVIEWS_DIR: buildPath(config.get('storage.previews')), @@ -259,6 +260,9 @@ const CONFIG = { get '480p' () { return config.get('transcoding.resolutions.480p') }, get '720p' () { return config.get('transcoding.resolutions.720p') }, get '1080p' () { return config.get('transcoding.resolutions.1080p') } + }, + HLS: { + get ENABLED () { return config.get('transcoding.hls.enabled') } } }, IMPORT: { @@ -590,6 +594,9 @@ const STATIC_PATHS = { TORRENTS: '/static/torrents/', WEBSEED: '/static/webseed/', REDUNDANCY: '/static/redundancy/', + PLAYLISTS: { + HLS: '/static/playlists/hls' + }, AVATARS: '/static/avatars/', VIDEO_CAPTIONS: '/static/video-captions/' } @@ -632,6 +639,9 @@ const CACHE = { } } +const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls') +const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') + const MEMOIZE_TTL = { OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours } @@ -709,6 +719,7 @@ updateWebserverUrls() export { API_VERSION, + HLS_REDUNDANCY_DIRECTORY, AVATARS_SIZE, ACCEPT_HEADERS, BCRYPT_SALT_SIZE, @@ -733,6 +744,7 @@ export { PRIVATE_RSA_KEY_SIZE, ROUTE_CACHE_LIFETIME, SORTABLE_COLUMNS, + HLS_PLAYLIST_DIRECTORY, FEEDS, JOB_TTL, NSFW_POLICY_TYPES, diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 84ad2079b..fe296142d 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist' import { ServerBlocklistModel } from '../models/server/server-blocklist' import { UserNotificationModel } from '../models/account/user-notification' import { UserNotificationSettingModel } from '../models/account/user-notification-setting' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) { AccountBlocklistModel, ServerBlocklistModel, UserNotificationModel, - UserNotificationSettingModel + UserNotificationSettingModel, + VideoStreamingPlaylistModel ]) // Check extensions exist in the database diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index b9a9da183..2b22e16fe 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { applicationExist, clientsExist, usersExist } from './checker-after-init' -import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' +import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' import { sequelizeTypescript } from './database' import { remove, ensureDir } from 'fs-extra' @@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () { tasks.push(ensureDir(dir)) } + // Playlist directories + tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY)) + return Promise.all(tasks) } diff --git a/server/initializers/migrations/0330-video-streaming-playlist.ts b/server/initializers/migrations/0330-video-streaming-playlist.ts new file mode 100644 index 000000000..c85a762ab --- /dev/null +++ b/server/initializers/migrations/0330-video-streaming-playlist.ts @@ -0,0 +1,51 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize +}): Promise { + + { + const query = ` + CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist" +( + "id" SERIAL, + "type" INTEGER NOT NULL, + "playlistUrl" VARCHAR(2000) NOT NULL, + "p2pMediaLoaderInfohashes" VARCHAR(255)[] NOT NULL, + "segmentsSha256Url" VARCHAR(255) NOT NULL, + "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, + "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL, + PRIMARY KEY ("id") +);` + await utils.sequelize.query(query) + } + + { + const data = { + type: Sequelize.INTEGER, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data) + } + + { + const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' + + 'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE' + + await utils.sequelize.query(query) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index f6f068b45..9a40414bb 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -1,11 +1,28 @@ -import { CacheFileObject } from '../../../shared/index' +import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index' import { VideoModel } from '../../models/video/video' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { Transaction } from 'sequelize' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { - const url = cacheFileObject.url + if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { + const url = cacheFileObject.url + + const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) + if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) + + return { + expiresOn: new Date(cacheFileObject.expires), + url: cacheFileObject.id, + fileUrl: url.href, + strategy: null, + videoStreamingPlaylistId: playlist.id, + actorId: byActor.id + } + } + + const url = cacheFileObject.url const videoFile = video.VideoFiles.find(f => { return f.resolution === url.height && f.fps === url.fps }) @@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject return { expiresOn: new Date(cacheFileObject.expires), url: cacheFileObject.id, - fileUrl: cacheFileObject.url.href, + fileUrl: url.href, strategy: null, videoFileId: videoFile.id, actorId: byActor.id diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index e3fca0a17..605aaba06 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -1,6 +1,6 @@ import { Transaction } from 'sequelize' import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' -import { VideoPrivacy } from '../../../../shared/models/videos' +import { Video, VideoPrivacy } from '../../../../shared/models/videos' import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' import { VideoAbuseModel } from '../../../models/video/video-abuse' @@ -39,17 +39,14 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } -async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { +async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) { logger.info('Creating job to send file cache of %s.', fileRedundancy.url) - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) - const redundancyObject = fileRedundancy.toActivityPubObject() - return sendVideoRelatedCreateActivity({ byActor, video, url: fileRedundancy.url, - object: redundancyObject + object: fileRedundancy.toActivityPubObject() }) } diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index bf1b6e117..8976fcbc8 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { logger.info('Creating job to undo cache file %s.', redundancyModel.url) - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) + const videoId = redundancyModel.getVideo().id + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index a68f03edf..839f66470 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts @@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { logger.info('Creating job to update cache file %s.', redundancyModel.url) - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id) const activityBuilder = (audience: ActivityAudience) => { const redundancyObject = redundancyModel.toActivityPubObject() diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 38f15448c..4229fe094 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts @@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video' import { VideoAbuseModel } from '../../models/video/video-abuse' import { VideoCommentModel } from '../../models/video/video-comment' import { VideoFileModel } from '../../models/video/video-file' +import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' function getVideoActivityPubUrl (video: VideoModel) { return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid @@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` } +function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) { + return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}` +} + function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id } @@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) { export { getVideoActivityPubUrl, + getVideoCacheStreamingPlaylistActivityPubUrl, getVideoChannelActivityPubUrl, getAccountActivityPubUrl, getVideoAbuseActivityPubUrl, diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index e1e523499..edd01234f 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird' import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' import * as request from 'request' -import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' +import { + ActivityIconObject, + ActivityPlaylistSegmentHashesObject, + ActivityPlaylistUrlObject, + ActivityUrlObject, + ActivityVideoUrlObject, + VideoState +} from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' @@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { Notifier } from '../notifier' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' +import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -263,6 +273,25 @@ async function updateVideoFromAP (options: { options.video.VideoFiles = await Promise.all(upsertTasks) } + { + const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) + const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) + + // Remove video files that do not exist anymore + const destroyTasks = options.video.VideoStreamingPlaylists + .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) + .map(f => f.destroy(sequelizeOptions)) + await Promise.all(destroyTasks) + + // Update or add other one + const upsertTasks = streamingPlaylistAttributes.map(a => { + return VideoStreamingPlaylistModel.upsert(a, { returning: true, transaction: t }) + .then(([ streamingPlaylist ]) => streamingPlaylist) + }) + + options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks) + } + { // Update Tags const tags = options.videoObject.tag.map(tag => tag.name) @@ -367,13 +396,25 @@ export { // --------------------------------------------------------------------------- -function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { +function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) const urlMediaType = url.mediaType || url.mimeType return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') } +function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { + const urlMediaType = url.mediaType || url.mimeType + + return urlMediaType === 'application/x-mpegURL' +} + +function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { + const urlMediaType = tag.mediaType || tag.mimeType + + return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' +} + async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { logger.debug('Adding remote video %s.', videoObject.id) @@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) await Promise.all(videoFilePromises) + const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) + const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) + await Promise.all(playlistPromises) + // Process tags - const tags = videoObject.tag.map(t => t.name) + const tags = videoObject.tag + .filter(t => t.type === 'Hashtag') + .map(t => t.name) const tagInstances = await TagModel.findOrCreateTags(tags, t) await videoCreated.$set('Tags', tagInstances, sequelizeOptions) @@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes ( } function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { - const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] + const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] if (fileUrls.length === 0) { throw new Error('Cannot find video files for ' + video.url) } - const attributes: VideoFileModel[] = [] + const attributes: FilteredModelAttributes[] = [] for (const fileUrl of fileUrls) { // Fetch associated magnet uri const magnet = videoObject.url.find(u => { @@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid size: fileUrl.size, videoId: video.id, fps: fileUrl.fps || -1 - } as VideoFileModel + } + + attributes.push(attribute) + } + + return attributes +} + +function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { + const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] + if (playlistUrls.length === 0) return [] + + const attributes: FilteredModelAttributes[] = [] + for (const playlistUrlObject of playlistUrls) { + const p2pMediaLoaderInfohashes = playlistUrlObject.tag + .filter(t => t.type === 'Infohash') + .map(t => t.name) + if (p2pMediaLoaderInfohashes.length === 0) { + logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject }) + continue + } + + const segmentsSha256UrlObject = playlistUrlObject.tag + .find(t => { + return isAPPlaylistSegmentHashesUrlObject(t) + }) as ActivityPlaylistSegmentHashesObject + if (!segmentsSha256UrlObject) { + logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) + continue + } + + const attribute = { + type: VideoStreamingPlaylistType.HLS, + playlistUrl: playlistUrlObject.href, + segmentsSha256Url: segmentsSha256UrlObject.href, + p2pMediaLoaderInfohashes, + videoId: video.id + } + attributes.push(attribute) } diff --git a/server/lib/hls.ts b/server/lib/hls.ts new file mode 100644 index 000000000..10db6c3c3 --- /dev/null +++ b/server/lib/hls.ts @@ -0,0 +1,110 @@ +import { VideoModel } from '../models/video/video' +import { basename, dirname, join } from 'path' +import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' +import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' +import { getVideoFileSize } from '../helpers/ffmpeg-utils' +import { sha256 } from '../helpers/core-utils' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' +import HLSDownloader from 'hlsdownloader' +import { logger } from '../helpers/logger' +import { parse } from 'url' + +async function updateMasterHLSPlaylist (video: VideoModel) { + const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] + const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) + + for (const file of video.VideoFiles) { + // If we did not generated a playlist for this resolution, skip + const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + if (await pathExists(filePlaylistPath) === false) continue + + const videoFilePath = video.getVideoFilePath(file) + + const size = await getVideoFileSize(videoFilePath) + + const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) + const resolution = `RESOLUTION=${size.width}x${size.height}` + + let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` + if (file.fps) line += ',FRAME-RATE=' + file.fps + + masterPlaylists.push(line) + masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + } + + await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') +} + +async function updateSha256Segments (video: VideoModel) { + const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + const files = await readdir(directory) + const json: { [filename: string]: string} = {} + + for (const file of files) { + if (file.endsWith('.ts') === false) continue + + const buffer = await readFile(join(directory, file)) + const filename = basename(file) + + json[filename] = sha256(buffer) + } + + const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + await outputJSON(outputPath, json) +} + +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { + let timer + + logger.info('Importing HLS playlist %s', playlistUrl) + + const params = { + playlistURL: playlistUrl, + destination: CONFIG.STORAGE.TMP_DIR + } + const downloader = new HLSDownloader(params) + + const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname)) + + return new Promise(async (res, rej) => { + downloader.startDownload(err => { + clearTimeout(timer) + + if (err) { + deleteTmpDirectory(hlsDestinationDir) + + return rej(err) + } + + move(hlsDestinationDir, destinationDir, { overwrite: true }) + .then(() => res()) + .catch(err => { + deleteTmpDirectory(hlsDestinationDir) + + return rej(err) + }) + }) + + timer = setTimeout(() => { + deleteTmpDirectory(hlsDestinationDir) + + return rej(new Error('HLS download timeout.')) + }, timeout) + + function deleteTmpDirectory (directory: string) { + remove(directory) + .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + } + }) +} + +// --------------------------------------------------------------------------- + +export { + updateMasterHLSPlaylist, + updateSha256Segments, + downloadPlaylistSegments +} + +// --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 217d666b6..7119ce0ca 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video' import { JobQueue } from '../job-queue' import { federateVideoIfNeeded } from '../../activitypub' import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { sequelizeTypescript } from '../../../initializers' +import { sequelizeTypescript, CONFIG } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' -import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' +import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' import { Notifier } from '../../notifier' export type VideoFilePayload = { videoUUID: string - isNewVideo?: boolean resolution?: VideoResolution + isNewVideo?: boolean isPortraitMode?: boolean + generateHlsPlaylist?: boolean } export type VideoFileImportPayload = { @@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) { return undefined } - // Transcoding in other resolution - if (payload.resolution) { + if (payload.generateHlsPlaylist) { + await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) + + await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) + } else if (payload.resolution) { // Transcoding in other resolution await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) - await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) + await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload) } else { await optimizeVideofile(video) - await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) + await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) } return video } -async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { +async function onHlsPlaylistGenerationSuccess (video: VideoModel) { + if (video === undefined) return undefined + + await sequelizeTypescript.transaction(async t => { + // Maybe the video changed in database, refresh it + let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) + // Video does not exist anymore + if (!videoDatabase) return undefined + + // If the video was not published, we consider it is a new one for other instances + await federateVideoIfNeeded(videoDatabase, false, t) + }) +} + +async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) { if (video === undefined) return undefined const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { @@ -96,9 +114,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { Notifier.Instance.notifyOnNewVideo(videoDatabase) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } + + await createHlsJobIfEnabled(payload) } -async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { +async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) { if (videoArg === undefined) return undefined // Outside the transaction (IO on disk) @@ -145,7 +165,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) } - await federateVideoIfNeeded(videoDatabase, isNewVideo, t) + await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) return { videoDatabase, videoPublished } }) @@ -155,6 +175,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } + + await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) } // --------------------------------------------------------------------------- @@ -163,3 +185,20 @@ export { processVideoFile, processVideoFileImport } + +// --------------------------------------------------------------------------- + +function createHlsJobIfEnabled (payload?: VideoFilePayload) { + // Generate HLS playlist? + if (payload && CONFIG.TRANSCODING.HLS.ENABLED) { + const hlsTranscodingPayload = { + videoUUID: payload.videoUUID, + resolution: payload.resolution, + isPortraitMode: payload.isPortraitMode, + + generateHlsPlaylist: true + } + + return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload }) + } +} diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f643ee226..1a48f2bd0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,5 +1,5 @@ import { AbstractScheduler } from './abstract-scheduler' -import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' +import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' import { logger } from '../../helpers/logger' import { VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' @@ -9,9 +9,19 @@ import { join } from 'path' import { move } from 'fs-extra' import { getServerActor } from '../../helpers/utils' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' -import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' +import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' import { removeVideoRedundancy } from '../redundancy' import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' +import { VideoModel } from '../../models/video/video' +import { downloadPlaylistSegments } from '../hls' + +type CandidateToDuplicate = { + redundancy: VideosRedundancy, + video: VideoModel, + files: VideoFileModel[], + streamingPlaylists: VideoStreamingPlaylistModel[] +} export class VideosRedundancyScheduler extends AbstractScheduler { @@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } protected async internalExecute () { - for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { - logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) + for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { + logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) try { - const videoToDuplicate = await this.findVideoToDuplicate(obj) + const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) if (!videoToDuplicate) continue - const videoFiles = videoToDuplicate.VideoFiles - videoFiles.forEach(f => f.Video = videoToDuplicate) + const candidateToDuplicate = { + video: videoToDuplicate, + redundancy: redundancyConfig, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + } - await this.purgeCacheIfNeeded(obj, videoFiles) + await this.purgeCacheIfNeeded(candidateToDuplicate) - if (await this.isTooHeavy(obj, videoFiles)) { + if (await this.isTooHeavy(candidateToDuplicate)) { logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) continue } - logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) + logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) - await this.createVideoRedundancy(obj, videoFiles) + await this.createVideoRedundancies(candidateToDuplicate) } catch (err) { - logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) + logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) } } @@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { for (const redundancyModel of expired) { try { - await this.extendsOrDeleteRedundancy(redundancyModel) + const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) + const candidate = { + redundancy: redundancyConfig, + video: null, + files: [], + streamingPlaylists: [] + } + + // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it + if (!redundancyConfig || await this.isTooHeavy(candidate)) { + logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) + await removeVideoRedundancy(redundancyModel) + } else { + await this.extendsRedundancy(redundancyModel) + } } catch (err) { - logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) + logger.error( + 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), + { err } + ) } } } - private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { - // Refresh the video, maybe it was deleted - const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url) - - if (!video) { - logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url) - - await redundancyModel.destroy() - return - } - + private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) + // Redundancy strategy disabled, remove our redundancy instead of extending expiration + if (!redundancy) await removeVideoRedundancy(redundancyModel) + await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) } @@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { - const serverActor = await getServerActor() + private async createVideoRedundancies (data: CandidateToDuplicate) { + const video = await this.loadAndRefreshVideo(data.video.url) + + if (!video) { + logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) - for (const file of filesToDuplicate) { - const video = await this.loadAndRefreshVideo(file.Video.url) + return + } + for (const file of data.files) { const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) if (existingRedundancy) { - await this.extendsOrDeleteRedundancy(existingRedundancy) + await this.extendsRedundancy(existingRedundancy) continue } - if (!video) { - logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) + await this.createVideoFileRedundancy(data.redundancy, video, file) + } + + for (const streamingPlaylist of data.streamingPlaylists) { + const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) + if (existingRedundancy) { + await this.extendsRedundancy(existingRedundancy) continue } - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) + await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) + } + } - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) + private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { + file.Video = video - const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) + const serverActor = await getServerActor() - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) - await move(tmpPath, destPath) + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) - const createdModel = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), - url: getVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), - strategy: redundancy.strategy, - videoFileId: file.id, - actorId: serverActor.id - }) - createdModel.VideoFile = file + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) - await sendCreateCacheFile(serverActor, createdModel) + const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) - } + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) + await move(tmpPath, destPath) + + const createdModel = await VideoRedundancyModel.create({ + expiresOn: this.buildNewExpiration(redundancy.minLifetime), + url: getVideoCacheFileActivityPubUrl(file), + fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), + strategy: redundancy.strategy, + videoFileId: file.id, + actorId: serverActor.id + }) + + createdModel.VideoFile = file + + await sendCreateCacheFile(serverActor, video, createdModel) + + logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) + } + + private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { + playlist.Video = video + + const serverActor = await getServerActor() + + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) + + const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) + await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) + + const createdModel = await VideoRedundancyModel.create({ + expiresOn: this.buildNewExpiration(redundancy.minLifetime), + url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), + fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL), + strategy: redundancy.strategy, + videoStreamingPlaylistId: playlist.id, + actorId: serverActor.id + }) + + createdModel.VideoStreamingPlaylist = playlist + + await sendCreateCacheFile(serverActor, video, createdModel) + + logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) } private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { @@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await sendUpdateCacheFile(serverActor, redundancy) } - private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { - while (this.isTooHeavy(redundancy, filesToDuplicate)) { + private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { + while (this.isTooHeavy(candidateToDuplicate)) { + const redundancy = candidateToDuplicate.redundancy const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) if (!toDelete) return @@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { - const maxSize = redundancy.size + private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { + const maxSize = candidateToDuplicate.redundancy.size - const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) - const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) + const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) + const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) return totalWillDuplicate > maxSize } @@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private buildEntryLogId (object: VideoRedundancyModel) { - return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` + if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` + + return `${object.VideoStreamingPlaylist.playlistUrl}` } - private getTotalFileSizes (files: VideoFileModel[]) { + private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size - return files.reduce(fileReducer, 0) + return files.reduce(fileReducer, 0) * playlists.length } private async loadAndRefreshVideo (videoUrl: string) { diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 4460f46e4..608badfef 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -1,11 +1,14 @@ -import { CONFIG } from '../initializers' +import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' import { extname, join } from 'path' import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' -import { copy, remove, move, stat } from 'fs-extra' +import { copy, ensureDir, move, remove, stat } from 'fs-extra' import { logger } from '../helpers/logger' import { VideoResolution } from '../../shared/models/videos' import { VideoFileModel } from '../models/video/video-file' import { VideoModel } from '../models/video/video' +import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' +import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR @@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi const transcodeOptions = { inputPath: videoInputPath, - outputPath: videoTranscodedPath + outputPath: videoTranscodedPath, + resolution: inputVideoFile.resolution } // Could be very long! @@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi } } -async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { +async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const extname = '.mp4' @@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR size: 0, videoId: video.id }) - const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) + const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) const transcodeOptions = { inputPath: videoInputPath, outputPath: videoOutputPath, resolution, - isPortraitMode + isPortraitMode: isPortrait } await transcode(transcodeOptions) @@ -84,6 +88,38 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR video.VideoFiles.push(newVideoFile) } +async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { + const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid)) + + const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile())) + const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) + + const transcodeOptions = { + inputPath: videoInputPath, + outputPath, + resolution, + isPortraitMode, + generateHlsPlaylist: true + } + + await transcode(transcodeOptions) + + await updateMasterHLSPlaylist(video) + await updateSha256Segments(video) + + const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) + + await VideoStreamingPlaylistModel.upsert({ + videoId: video.id, + playlistUrl, + segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), + p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles), + + type: VideoStreamingPlaylistType.HLS + }) +} + async function importVideoFile (video: VideoModel, inputFilePath: string) { const { videoFileResolution } = await getVideoFileResolution(inputFilePath) const { size } = await stat(inputFilePath) @@ -125,6 +161,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) { } export { + generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile, importVideoFile diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts index c72ab78b2..329322509 100644 --- a/server/middlewares/validators/redundancy.ts +++ b/server/middlewares/validators/redundancy.ts @@ -13,7 +13,7 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow' import { SERVER_ACTOR_NAME } from '../../initializers' import { ServerModel } from '../../models/server/server' -const videoRedundancyGetValidator = [ +const videoFileRedundancyGetValidator = [ param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), param('resolution') .customSanitizer(toIntOrNull) @@ -24,7 +24,7 @@ const videoRedundancyGetValidator = [ .custom(exists).withMessage('Should have a valid fps'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) + logger.debug('Checking videoFileRedundancyGetValidator parameters', { parameters: req.params }) if (areValidationErrors(req, res)) return if (!await isVideoExist(req.params.videoId, res)) return @@ -38,7 +38,31 @@ const videoRedundancyGetValidator = [ res.locals.videoFile = videoFile const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) - if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) + if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' }) + res.locals.videoRedundancy = videoRedundancy + + return next() + } +] + +const videoPlaylistRedundancyGetValidator = [ + param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), + param('streamingPlaylistType').custom(exists).withMessage('Should have a valid streaming playlist type'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoPlaylistRedundancyGetValidator parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + if (!await isVideoExist(req.params.videoId, res)) return + + const video: VideoModel = res.locals.video + const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p === req.params.streamingPlaylistType) + + if (!videoStreamingPlaylist) return res.status(404).json({ error: 'Video playlist not found.' }) + res.locals.videoStreamingPlaylist = videoStreamingPlaylist + + const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id) + if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' }) res.locals.videoRedundancy = videoRedundancy return next() @@ -75,6 +99,7 @@ const updateServerRedundancyValidator = [ // --------------------------------------------------------------------------- export { - videoRedundancyGetValidator, + videoFileRedundancyGetValidator, + videoPlaylistRedundancyGetValidator, updateServerRedundancyValidator } diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 8f2ef2d9a..b722bed14 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -28,6 +28,7 @@ import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' import * as Bluebird from 'bluebird' import * as Sequelize from 'sequelize' +import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -38,7 +39,17 @@ export enum ScopeNames { include: [ { model: () => VideoFileModel, - required: true, + required: false, + include: [ + { + model: () => VideoModel, + required: true + } + ] + }, + { + model: () => VideoStreamingPlaylistModel, + required: false, include: [ { model: () => VideoModel, @@ -97,12 +108,24 @@ export class VideoRedundancyModel extends Model { @BelongsTo(() => VideoFileModel, { foreignKey: { - allowNull: false + allowNull: true }, onDelete: 'cascade' }) VideoFile: VideoFileModel + @ForeignKey(() => VideoStreamingPlaylistModel) + @Column + videoStreamingPlaylistId: number + + @BelongsTo(() => VideoStreamingPlaylistModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + VideoStreamingPlaylist: VideoStreamingPlaylistModel + @ForeignKey(() => ActorModel) @Column actorId: number @@ -119,13 +142,25 @@ export class VideoRedundancyModel extends Model { static async removeFile (instance: VideoRedundancyModel) { if (!instance.isOwned()) return - const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) + if (instance.videoFileId) { + const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) - const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` - logger.info('Removing duplicated video file %s.', logIdentifier) + const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` + logger.info('Removing duplicated video file %s.', logIdentifier) - videoFile.Video.removeFile(videoFile, true) - .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) + videoFile.Video.removeFile(videoFile, true) + .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) + } + + if (instance.videoStreamingPlaylistId) { + const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId) + + const videoUUID = videoStreamingPlaylist.Video.uuid + logger.info('Removing duplicated video streaming playlist %s.', videoUUID) + + videoStreamingPlaylist.Video.removeStreamingPlaylist(true) + .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) + } return undefined } @@ -143,6 +178,19 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } + static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) { + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + videoStreamingPlaylistId + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + static loadByUrl (url: string, transaction?: Sequelize.Transaction) { const query = { where: { @@ -191,7 +239,7 @@ export class VideoRedundancyModel extends Model { const ids = rows.map(r => r.id) const id = sample(ids) - return VideoModel.loadWithFile(id, undefined, !isTestInstance()) + return VideoModel.loadWithFiles(id, undefined, !isTestInstance()) } static async findMostViewToDuplicate (randomizedFactor: number) { @@ -333,40 +381,44 @@ export class VideoRedundancyModel extends Model { static async listLocalOfServer (serverId: number) { const actor = await getServerActor() - - const query = { - where: { - actorId: actor.id - }, + const buildVideoInclude = () => ({ + model: VideoModel, + required: true, include: [ { - model: VideoFileModel, + attributes: [], + model: VideoChannelModel.unscoped(), required: true, include: [ { - model: VideoModel, + attributes: [], + model: ActorModel.unscoped(), required: true, - include: [ - { - attributes: [], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ActorModel.unscoped(), - required: true, - where: { - serverId - } - } - ] - } - ] + where: { + serverId + } } ] } ] + }) + + const query = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoFileModel, + required: false, + include: [ buildVideoInclude() ] + }, + { + model: VideoStreamingPlaylistModel, + required: false, + include: [ buildVideoInclude() ] + } + ] } return VideoRedundancyModel.findAll(query) @@ -403,11 +455,32 @@ export class VideoRedundancyModel extends Model { })) } + getVideo () { + if (this.VideoFile) return this.VideoFile.Video + + return this.VideoStreamingPlaylist.Video + } + isOwned () { return !!this.strategy } toActivityPubObject (): CacheFileObject { + if (this.VideoStreamingPlaylist) { + return { + id: this.url, + type: 'CacheFile' as 'CacheFile', + object: this.VideoStreamingPlaylist.Video.url, + expires: this.expiresOn.toISOString(), + url: { + type: 'Link', + mimeType: 'application/x-mpegURL', + mediaType: 'application/x-mpegURL', + href: this.fileUrl + } + } + } + return { id: this.url, type: 'CacheFile' as 'CacheFile', @@ -431,7 +504,7 @@ export class VideoRedundancyModel extends Model { const notIn = Sequelize.literal( '(' + - `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + + `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + ')' ) diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 1f1b76c1e..7d1e371b9 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -62,7 +62,7 @@ export class VideoFileModel extends Model { extname: string @AllowNull(false) - @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) + @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) @Column infoHash: string @@ -86,14 +86,14 @@ export class VideoFileModel extends Model { @HasMany(() => VideoRedundancyModel, { foreignKey: { - allowNull: false + allowNull: true }, onDelete: 'CASCADE', hooks: true }) RedundancyVideos: VideoRedundancyModel[] - static isInfohashExists (infoHash: string) { + static doesInfohashExist (infoHash: string) { const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' const options = { type: Sequelize.QueryTypes.SELECT, diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index de0747f22..e49dbee30 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -1,7 +1,12 @@ import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoModel } from './video' import { VideoFileModel } from './video-file' -import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' +import { + ActivityPlaylistInfohashesObject, + ActivityPlaylistSegmentHashesObject, + ActivityUrlObject, + VideoTorrentObject +} from '../../../shared/models/activitypub/objects' import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' import { VideoCaptionModel } from './video-caption' import { @@ -11,6 +16,8 @@ import { getVideoSharesActivityPubUrl } from '../../lib/activitypub' import { isArray } from '../../helpers/custom-validators/misc' +import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist' export type VideoFormattingJSONOptions = { completeDescription?: boolean @@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { } }) + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + const tags = video.Tags ? video.Tags.map(t => t.name) : [] + + const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) + const detailsJson = { support: video.support, descriptionPath: video.getDescriptionAPIPath(), @@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { id: video.state, label: VideoModel.getStateLabel(video.state) }, - files: [] + + trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs), + + files: [], + streamingPlaylists } // Format and sort video files @@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { return Object.assign(formattedJson, detailsJson) } +function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] { + if (isArray(playlists) === false) return [] + + return playlists + .map(playlist => { + const redundancies = isArray(playlist.RedundancyVideos) + ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) + : [] + + return { + id: playlist.id, + type: playlist.type, + playlistUrl: playlist.playlistUrl, + segmentsSha256Url: playlist.segmentsSha256Url, + redundancies + } as VideoStreamingPlaylist + }) +} + function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() @@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { }) } + for (const playlist of (video.VideoStreamingPlaylists || [])) { + let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[] + + tag = playlist.p2pMediaLoaderInfohashes + .map(i => ({ type: 'Infohash' as 'Infohash', name: i })) + tag.push({ + type: 'Link', + name: 'sha256', + mimeType: 'application/json' as 'application/json', + mediaType: 'application/json' as 'application/json', + href: playlist.segmentsSha256Url + }) + + url.push({ + type: 'Link', + mimeType: 'application/x-mpegURL' as 'application/x-mpegURL', + mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', + href: playlist.playlistUrl, + tag + }) + } + // Add video url too url.push({ type: 'Link', diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts new file mode 100644 index 000000000..bce537781 --- /dev/null +++ b/server/models/video/video-streaming-playlist.ts @@ -0,0 +1,154 @@ +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' +import { throwIfNotValid } from '../utils' +import { VideoModel } from './video' +import * as Sequelize from 'sequelize' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' +import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers' +import { VideoFileModel } from './video-file' +import { join } from 'path' +import { sha1 } from '../../helpers/core-utils' +import { isArrayOf } from '../../helpers/custom-validators/misc' + +@Table({ + tableName: 'videoStreamingPlaylist', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'videoId', 'type' ], + unique: true + }, + { + fields: [ 'p2pMediaLoaderInfohashes' ], + using: 'gin' + } + ] +}) +export class VideoStreamingPlaylistModel extends Model { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column + type: VideoStreamingPlaylistType + + @AllowNull(false) + @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) + playlistUrl: string + + @AllowNull(false) + @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes')) + @Column(DataType.ARRAY(DataType.STRING)) + p2pMediaLoaderInfohashes: string[] + + @AllowNull(false) + @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) + @Column + segmentsSha256Url: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: VideoModel + + @HasMany(() => VideoRedundancyModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE', + hooks: true + }) + RedundancyVideos: VideoRedundancyModel[] + + static doesInfohashExist (infoHash: string) { + const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1' + const options = { + type: Sequelize.QueryTypes.SELECT, + bind: { infoHash }, + raw: true + } + + return VideoModel.sequelize.query(query, options) + .then(results => { + return results.length === 1 + }) + } + + static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) { + const hashes: string[] = [] + + // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97 + for (let i = 0; i < videoFiles.length; i++) { + hashes.push(sha1(`1${playlistUrl}+V${i}`)) + } + + return hashes + } + + static loadWithVideo (id: number) { + const options = { + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + } + + return VideoStreamingPlaylistModel.findById(id, options) + } + + static getHlsPlaylistFilename (resolution: number) { + return resolution + '.m3u8' + } + + static getMasterHlsPlaylistFilename () { + return 'master.m3u8' + } + + static getHlsSha256SegmentsFilename () { + return 'segments-sha256.json' + } + + static getHlsMasterPlaylistStaticPath (videoUUID: string) { + return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) + } + + static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) { + return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) + } + + static getHlsSha256SegmentsStaticPath (videoUUID: string) { + return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + } + + getStringType () { + if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' + + return 'unknown' + } + + getVideoRedundancyUrl (baseUrlHttp: string) { + return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid + } + + hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) { + return this.type === other.type && + this.videoId === other.videoId + } +} diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 80a6c7832..702260772 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -52,7 +52,7 @@ import { ACTIVITY_PUB, API_VERSION, CONFIG, - CONSTRAINTS_FIELDS, + CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY, PREVIEWS_SIZE, REMOTE_SCHEME, STATIC_DOWNLOAD_PATHS, @@ -95,6 +95,7 @@ import * as validator from 'validator' import { UserVideoHistoryModel } from '../account/user-video-history' import { UserModel } from '../account/user' import { VideoImportModel } from './video-import' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -159,7 +160,9 @@ export enum ScopeNames { WITH_FILES = 'WITH_FILES', WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', WITH_BLACKLISTED = 'WITH_BLACKLISTED', - WITH_USER_HISTORY = 'WITH_USER_HISTORY' + WITH_USER_HISTORY = 'WITH_USER_HISTORY', + WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', + WITH_USER_ID = 'WITH_USER_ID' } type ForAPIOptions = { @@ -463,6 +466,22 @@ type AvailableForListIDsOptions = { return query }, + [ ScopeNames.WITH_USER_ID ]: { + include: [ + { + attributes: [ 'accountId' ], + model: () => VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'userId' ], + model: () => AccountModel.unscoped(), + required: true + } + ] + } + ] + }, [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { include: [ { @@ -527,22 +546,55 @@ type AvailableForListIDsOptions = { } ] }, - [ ScopeNames.WITH_FILES ]: { - include: [ - { - model: () => VideoFileModel.unscoped(), - // FIXME: typings - [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join - required: false, - include: [ - { - attributes: [ 'fileUrl' ], - model: () => VideoRedundancyModel.unscoped(), - required: false - } - ] - } - ] + [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoFileModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } + }, + [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + // FIXME: typings + [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join + required: false, + include: subInclude + } + ] + } }, [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { include: [ @@ -722,6 +774,16 @@ export class VideoModel extends Model { }) VideoFiles: VideoFileModel[] + @HasMany(() => VideoStreamingPlaylistModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + hooks: true, + onDelete: 'cascade' + }) + VideoStreamingPlaylists: VideoStreamingPlaylistModel[] + @HasMany(() => VideoShareModel, { foreignKey: { name: 'videoId', @@ -847,6 +909,9 @@ export class VideoModel extends Model { tasks.push(instance.removeFile(file)) tasks.push(instance.removeTorrent(file)) }) + + // Remove playlists file + tasks.push(instance.removeStreamingPlaylist()) } // Do not wait video deletion because we could be in a transaction @@ -858,10 +923,6 @@ export class VideoModel extends Model { return undefined } - static list () { - return VideoModel.scope(ScopeNames.WITH_FILES).findAll() - } - static listLocal () { const query = { where: { @@ -869,7 +930,7 @@ export class VideoModel extends Model { } } - return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query) } static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { @@ -1200,6 +1261,16 @@ export class VideoModel extends Model { return VideoModel.findOne(options) } + static loadWithRights (id: number | string, t?: Sequelize.Transaction) { + const where = VideoModel.buildWhereIdOrUUID(id) + const options = { + where, + transaction: t + } + + return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options) + } + static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { const where = VideoModel.buildWhereIdOrUUID(id) @@ -1212,8 +1283,8 @@ export class VideoModel extends Model { return VideoModel.findOne(options) } - static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { - return VideoModel.scope(ScopeNames.WITH_FILES) + static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) { + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]) .findById(id, { transaction: t, logging }) } @@ -1224,9 +1295,7 @@ export class VideoModel extends Model { } } - return VideoModel - .scope([ ScopeNames.WITH_FILES ]) - .findOne(options) + return VideoModel.findOne(options) } static loadByUrl (url: string, transaction?: Sequelize.Transaction) { @@ -1248,7 +1317,11 @@ export class VideoModel extends Model { transaction } - return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) + return VideoModel.scope([ + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ]).findOne(query) } static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { @@ -1263,9 +1336,37 @@ export class VideoModel extends Model { const scopes = [ ScopeNames.WITH_TAGS, ScopeNames.WITH_BLACKLISTED, + ScopeNames.WITH_ACCOUNT_DETAILS, + ScopeNames.WITH_SCHEDULED_UPDATE, ScopeNames.WITH_FILES, + ScopeNames.WITH_STREAMING_PLAYLISTS + ] + + if (userId) { + scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings + } + + return VideoModel + .scope(scopes) + .findOne(options) + } + + static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) { + const where = VideoModel.buildWhereIdOrUUID(id) + + const options = { + order: [ [ 'Tags', 'name', 'ASC' ] ], + where, + transaction: t + } + + const scopes = [ + ScopeNames.WITH_TAGS, + ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_ACCOUNT_DETAILS, - ScopeNames.WITH_SCHEDULED_UPDATE + ScopeNames.WITH_SCHEDULED_UPDATE, + { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings + { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings ] if (userId) { @@ -1612,6 +1713,14 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } + removeStreamingPlaylist (isRedundancy = false) { + const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY + + const filePath = join(baseDir, this.uuid) + return remove(filePath) + .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) + } + isOutdated () { if (this.isOwned()) return false @@ -1646,7 +1755,7 @@ export class VideoModel extends Model { generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { const xs = this.getTorrentUrl(videoFile, baseUrlHttp) - const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs) let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] const redundancies = videoFile.RedundancyVideos @@ -1663,6 +1772,10 @@ export class VideoModel extends Model { return magnetUtil.encode(magnetHash) } + getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { + return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + } + getThumbnailUrl (baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() } @@ -1686,4 +1799,8 @@ export class VideoModel extends Model { getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } + + getBandwidthBits (videoFile: VideoFileModel) { + return Math.ceil((videoFile.size * 8) / this.duration) + } } diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 4038ecbf0..07de2b5a5 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -65,6 +65,9 @@ describe('Test config API validators', function () { '480p': true, '720p': false, '1080p': false + }, + hls: { + enabled: false } }, import: { diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 9d3ce8153..5b99309fb 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -17,7 +17,7 @@ import { viewVideo, wait, waitUntilLog, - checkVideoFilesWereRemoved, removeVideo, getVideoWithToken + checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer } from '../../../../shared/utils' import { waitJobs } from '../../../../shared/utils/server/jobs' @@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { const config = { + transcoding: { + hls: { + enabled: true + } + }, redundancy: { videos: { check_interval: '5 seconds', @@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: await waitJobs(servers) } -async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { +async function check1WebSeed (videoUUID?: string) { if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ @@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str ] for (const server of servers) { - { - // With token to avoid issues with video follow constraints - const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) + // With token to avoid issues with video follow constraints + const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) - const video: VideoDetails = res.body - for (const f of video.files) { - checkMagnetWebseeds(f, webseeds, server) - } + const video: VideoDetails = res.body + for (const f of video.files) { + checkMagnetWebseeds(f, webseeds, server) } } } -async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - const stat = data.videosRedundancy[0] - - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(204800) - expect(stat.totalUsed).to.be.at.least(1).and.below(204801) - expect(stat.totalVideoFiles).to.equal(4) - expect(stat.totalVideos).to.equal(1) -} - -async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(204800) - expect(stat.totalUsed).to.equal(0) - expect(stat.totalVideoFiles).to.equal(0) - expect(stat.totalVideos).to.equal(0) -} - -async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { +async function check2Webseeds (videoUUID?: string) { if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ @@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st await makeGetRequest({ url: servers[1].url, statusCodeExpected: 200, - path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, contentType: null }) } @@ -174,6 +149,81 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st } } +async function check0PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + for (const server of servers) { + // With token to avoid issues with video follow constraints + const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) + const video: VideoDetails = res.body + + expect(video.streamingPlaylists).to.be.an('array') + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) + } +} + +async function check1PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video: VideoDetails = res.body + + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) + + const redundancy = video.streamingPlaylists[0].redundancies[0] + + expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) + } + + await makeGetRequest({ + url: servers[0].url, + statusCodeExpected: 200, + path: `/static/redundancy/hls/${videoUUID}/360_000.ts`, + contentType: null + }) + + for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { + const files = await readdir(join(root(), directory, videoUUID)) + expect(files).to.have.length.at.least(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined + expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined + } + } +} + +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] + + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(204800) + expect(stat.totalUsed).to.be.at.least(1).and.below(204801) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) +} + +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(204800) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) +} + async function enableRedundancyOnServer1 () { await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) @@ -220,7 +270,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -229,27 +280,29 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) it('Should undo redundancy on server 1 and remove duplicated videos', async function () { - this.timeout(40000) + this.timeout(80000) await disableRedundancyOnServer1() await waitJobs(servers) await wait(5000) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ]) }) after(function () { @@ -267,7 +320,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -276,25 +330,27 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) it('Should unfollow on server 1 and remove duplicated videos', async function () { - this.timeout(40000) + this.timeout(80000) await unfollow(servers[0].url, servers[0].accessToken, servers[1]) await waitJobs(servers) await wait(5000) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) }) @@ -314,7 +370,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -323,18 +380,19 @@ describe('Test videos redundancy', function () { }) it('Should still have 1 webseed on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) await wait(15000) await waitJobs(servers) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) it('Should view 2 times the first video to have > min_views config', async function () { - this.timeout(40000) + this.timeout(80000) await viewVideo(servers[ 0 ].url, video1Server2UUID) await viewVideo(servers[ 2 ].url, video1Server2UUID) @@ -344,13 +402,14 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) @@ -405,7 +464,7 @@ describe('Test videos redundancy', function () { }) it('Should still have 2 webseeds after 10 seconds', async function () { - this.timeout(40000) + this.timeout(80000) await wait(10000) @@ -420,7 +479,7 @@ describe('Test videos redundancy', function () { }) it('Should stop server 1 and expire video redundancy', async function () { - this.timeout(40000) + this.timeout(80000) killallServers([ servers[0] ]) @@ -446,10 +505,11 @@ describe('Test videos redundancy', function () { await enableRedundancyOnServer1() await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) @@ -467,8 +527,10 @@ describe('Test videos redundancy', function () { await wait(1000) try { - await check1WebSeed(strategy, video1Server2UUID) - await check2Webseeds(strategy, video2Server2UUID) + await check1WebSeed(video1Server2UUID) + await check0PlaylistRedundancies(video1Server2UUID) + await check2Webseeds(video2Server2UUID) + await check1PlaylistRedundancies(video2Server2UUID) checked = true } catch { @@ -477,6 +539,26 @@ describe('Test videos redundancy', function () { } }) + it('Should disable strategy and remove redundancies', async function () { + this.timeout(80000) + + await waitJobs(servers) + + killallServers([ servers[ 0 ] ]) + await reRunServer(servers[ 0 ], { + redundancy: { + videos: { + check_interval: '1 second', + strategies: [] + } + } + }) + + await waitJobs(servers) + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ]) + }) + after(function () { return cleanServers() }) diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index bebfc7398..0dfe6e4fe 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) { expect(data.transcoding.resolutions['480p']).to.be.true expect(data.transcoding.resolutions['720p']).to.be.true expect(data.transcoding.resolutions['1080p']).to.be.true + expect(data.transcoding.hls.enabled).to.be.true + expect(data.import.videos.http.enabled).to.be.true expect(data.import.videos.torrent.enabled).to.be.true } @@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.transcoding.resolutions['480p']).to.be.true expect(data.transcoding.resolutions['720p']).to.be.false expect(data.transcoding.resolutions['1080p']).to.be.false + expect(data.transcoding.hls.enabled).to.be.false expect(data.import.videos.http.enabled).to.be.false expect(data.import.videos.torrent.enabled).to.be.false @@ -205,6 +208,9 @@ describe('Test config', function () { '480p': true, '720p': false, '1080p': false + }, + hls: { + enabled: false } }, import: { diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 97f467aae..a501a80b2 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -8,6 +8,7 @@ import './video-change-ownership' import './video-channels' import './video-comments' import './video-description' +import './video-hls' import './video-imports' import './video-nsfw' import './video-privacy' diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts new file mode 100644 index 000000000..71d863b12 --- /dev/null +++ b/server/tests/api/videos/video-hls.ts @@ -0,0 +1,145 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { + checkDirectoryIsEmpty, + checkTmpIsEmpty, + doubleFollow, + flushAndRunMultipleServers, + flushTests, + getPlaylist, + getSegment, + getSegmentSha256, + getVideo, + killallServers, + removeVideo, + ServerInfo, + setAccessTokensToServers, + updateVideo, + uploadVideo, + waitJobs +} from '../../../../shared/utils' +import { VideoDetails } from '../../../../shared/models/videos' +import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' +import { sha256 } from '../../../helpers/core-utils' +import { join } from 'path' + +const expect = chai.expect + +async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) { + const resolutions = [ 240, 360, 480, 720 ] + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const videoDetails: VideoDetails = res.body + + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + + const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + expect(hlsPlaylist).to.not.be.undefined + + { + const res2 = await getPlaylist(hlsPlaylist.playlistUrl) + + const masterPlaylist = res2.text + + expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25') + + for (const resolution of resolutions) { + expect(masterPlaylist).to.contain(`${resolution}.m3u8`) + } + } + + { + for (const resolution of resolutions) { + const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`) + + const subPlaylist = res2.text + expect(subPlaylist).to.contain(resolution + '_000.ts') + } + } + + { + for (const resolution of resolutions) { + + const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`) + + const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) + + const sha256Server = resSha.body[ resolution + '_000.ts' ] + expect(sha256(res2.body)).to.equal(sha256Server) + } + } + } +} + +describe('Test HLS videos', function () { + let servers: ServerInfo[] = [] + let videoUUID = '' + + before(async function () { + this.timeout(120000) + + servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } }) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + it('Should upload a video and transcode it to HLS', async function () { + this.timeout(120000) + + { + const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' }) + videoUUID = res.body.video.uuid + } + + await waitJobs(servers) + + await checkHlsPlaylist(servers, videoUUID) + }) + + it('Should update the video', async function () { + await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' }) + + await waitJobs(servers) + + await checkHlsPlaylist(servers, videoUUID) + }) + + it('Should delete the video', async function () { + await removeVideo(servers[0].url, servers[0].accessToken, videoUUID) + + await waitJobs(servers) + + for (const server of servers) { + await getVideo(server.url, videoUUID, 404) + } + }) + + it('Should have the playlists/segment deleted from the disk', async function () { + for (const server of servers) { + await checkDirectoryIsEmpty(server, 'videos') + await checkDirectoryIsEmpty(server, join('playlists', 'hls')) + } + }) + + it('Should have an empty tmp directory', async function () { + for (const server of servers) { + await checkTmpIsEmpty(server) + } + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts index 811ea6a9f..d38bb4331 100644 --- a/server/tests/cli/update-host.ts +++ b/server/tests/cli/update-host.ts @@ -86,6 +86,13 @@ describe('Test update host scripts', function () { const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) + + const res = await getVideo(server.url, video.uuid) + const videoDetails: VideoDetails = res.body + + expect(videoDetails.trackerUrls[0]).to.include(server.host) + expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host) + expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host) } }) @@ -100,7 +107,7 @@ describe('Test update host scripts', function () { } }) - it('Should have update accounts url', async function () { + it('Should have updated accounts url', async function () { const res = await getAccountsList(server.url) expect(res.body.total).to.equal(3) @@ -112,7 +119,7 @@ describe('Test update host scripts', function () { } }) - it('Should update torrent hosts', async function () { + it('Should have updated torrent hosts', async function () { this.timeout(30000) const res = await getVideosList(server.url) -- cgit v1.2.3 From 4c280004ce62bf11ddb091854c28f1e1d54a54d6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 7 Feb 2019 15:08:19 +0100 Subject: Use a single file instead of segments for HLS --- server/helpers/ffmpeg-utils.ts | 12 ++- server/helpers/requests.ts | 2 +- server/helpers/utils.ts | 1 - server/lib/activitypub/actor.ts | 4 +- server/lib/hls.ts | 136 +++++++++++++++++------- server/lib/video-transcoding.ts | 5 +- server/models/video/video-streaming-playlist.ts | 4 + server/tests/api/redundancy/redundancy.ts | 22 ++-- server/tests/api/videos/video-hls.ts | 16 +-- 9 files changed, 132 insertions(+), 70 deletions(-) (limited to 'server') diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 5ad8ed48e..133b1b03b 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -122,7 +122,9 @@ type TranscodeOptions = { resolution: VideoResolution isPortraitMode?: boolean - generateHlsPlaylist?: boolean + hlsPlaylist?: { + videoFilename: string + } } function transcode (options: TranscodeOptions) { @@ -161,14 +163,16 @@ function transcode (options: TranscodeOptions) { command = command.withFPS(fps) } - if (options.generateHlsPlaylist) { - const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts` + if (options.hlsPlaylist) { + const videoPath = `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` command = command.outputOption('-hls_time 4') .outputOption('-hls_list_size 0') .outputOption('-hls_playlist_type vod') - .outputOption('-hls_segment_filename ' + segmentFilename) + .outputOption('-hls_segment_filename ' + videoPath) + .outputOption('-hls_segment_type fmp4') .outputOption('-f hls') + .outputOption('-hls_flags single_file') } command diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 3fc776f1a..5c6dc5e19 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -7,7 +7,7 @@ import { join } from 'path' function doRequest ( requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } -): Bluebird<{ response: request.RequestResponse, body: any }> { +): Bluebird<{ response: request.RequestResponse, body: T }> { if (requestOptions.activityPub === true) { if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 3c3406e38..cb0e823c5 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -7,7 +7,6 @@ import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' import { remove } from 'fs-extra' import * as memoizee from 'memoizee' -import { isArray } from './custom-validators/misc' function deleteFileAsync (path: string) { remove(path) diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 8215840da..a3f379b76 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -355,10 +355,10 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe logger.info('Fetching remote actor %s.', actorUrl) - const requestResult = await doRequest(options) + const requestResult = await doRequest(options) normalizeActor(requestResult.body) - const actorJSON: ActivityPubActor = requestResult.body + const actorJSON = requestResult.body if (isActorObjectValid(actorJSON) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON }) return { result: undefined, statusCode: requestResult.response.statusCode } diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 10db6c3c3..3575981f4 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,13 +1,14 @@ import { VideoModel } from '../models/video/video' -import { basename, dirname, join } from 'path' -import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' -import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' +import { basename, join, dirname } from 'path' +import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' +import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' import { getVideoFileSize } from '../helpers/ffmpeg-utils' import { sha256 } from '../helpers/core-utils' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import HLSDownloader from 'hlsdownloader' import { logger } from '../helpers/logger' -import { parse } from 'url' +import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' +import { generateRandomString } from '../helpers/utils' +import { flatten, uniq } from 'lodash' async function updateMasterHLSPlaylist (video: VideoModel) { const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) @@ -37,66 +38,119 @@ async function updateMasterHLSPlaylist (video: VideoModel) { } async function updateSha256Segments (video: VideoModel) { - const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) - const files = await readdir(directory) - const json: { [filename: string]: string} = {} + const json: { [filename: string]: { [range: string]: string } } = {} + + const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + + // For all the resolutions available for this video + for (const file of video.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} + + const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) + const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) - for (const file of files) { - if (file.endsWith('.ts') === false) continue + // Maybe the playlist is not generated for this resolution yet + if (!await pathExists(playlistPath)) continue - const buffer = await readFile(join(directory, file)) - const filename = basename(file) + const playlistContent = await readFile(playlistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) - json[filename] = sha256(buffer) + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) + + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) + + const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) + json[videoFilename] = rangeHashes } - const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) await outputJSON(outputPath, json) } -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { - let timer +function getRangesFromPlaylist (playlistContent: string) { + const ranges: { offset: number, length: number }[] = [] + const lines = playlistContent.split('\n') + const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ - logger.info('Importing HLS playlist %s', playlistUrl) + for (const line of lines) { + const captured = regex.exec(line) - const params = { - playlistURL: playlistUrl, - destination: CONFIG.STORAGE.TMP_DIR + if (captured) { + ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) + } } - const downloader = new HLSDownloader(params) - - const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname)) - return new Promise(async (res, rej) => { - downloader.startDownload(err => { - clearTimeout(timer) + return ranges +} - if (err) { - deleteTmpDirectory(hlsDestinationDir) +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { + let timer - return rej(err) - } + logger.info('Importing HLS playlist %s', playlistUrl) - move(hlsDestinationDir, destinationDir, { overwrite: true }) - .then(() => res()) - .catch(err => { - deleteTmpDirectory(hlsDestinationDir) + return new Promise(async (res, rej) => { + const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) - return rej(err) - }) - }) + await ensureDir(tmpDirectory) timer = setTimeout(() => { - deleteTmpDirectory(hlsDestinationDir) + deleteTmpDirectory(tmpDirectory) return rej(new Error('HLS download timeout.')) }, timeout) - function deleteTmpDirectory (directory: string) { - remove(directory) - .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + try { + // Fetch master playlist + const subPlaylistUrls = await fetchUniqUrls(playlistUrl) + + const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) + const fileUrls = uniq(flatten(await Promise.all(subRequests))) + + logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) + + for (const fileUrl of fileUrls) { + const destPath = join(tmpDirectory, basename(fileUrl)) + + await doRequestAndSaveToFile({ uri: fileUrl }, destPath) + } + + clearTimeout(timer) + + await move(tmpDirectory, destinationDir, { overwrite: true }) + + return res() + } catch (err) { + deleteTmpDirectory(tmpDirectory) + + return rej(err) } }) + + function deleteTmpDirectory (directory: string) { + remove(directory) + .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + } + + async function fetchUniqUrls (playlistUrl: string) { + const { body } = await doRequest({ uri: playlistUrl }) + + if (!body) return [] + + const urls = body.split('\n') + .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) + .map(url => { + if (url.startsWith('http://') || url.startsWith('https://')) return url + + return `${dirname(playlistUrl)}/${url}` + }) + + return uniq(urls) + } } // --------------------------------------------------------------------------- diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 608badfef..086b860a2 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -100,7 +100,10 @@ async function generateHlsPlaylist (video: VideoModel, resolution: VideoResoluti outputPath, resolution, isPortraitMode, - generateHlsPlaylist: true + + hlsPlaylist: { + videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) + } } await transcode(transcodeOptions) diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index bce537781..bf6f7b0c4 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -125,6 +125,10 @@ export class VideoStreamingPlaylistModel extends Model f === `${resolution}_000.ts`)).to.not.be.undefined - expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined + const filename = `${videoUUID}-${resolution}-fragmented.mp4` + + expect(files.find(f => f === filename)).to.not.be.undefined } } } diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 71d863b12..a1214bad1 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -4,13 +4,12 @@ import * as chai from 'chai' import 'mocha' import { checkDirectoryIsEmpty, + checkSegmentHash, checkTmpIsEmpty, doubleFollow, flushAndRunMultipleServers, flushTests, getPlaylist, - getSegment, - getSegmentSha256, getVideo, killallServers, removeVideo, @@ -22,7 +21,6 @@ import { } from '../../../../shared/utils' import { VideoDetails } from '../../../../shared/models/videos' import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' -import { sha256 } from '../../../helpers/core-utils' import { join } from 'path' const expect = chai.expect @@ -56,19 +54,15 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) { const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`) const subPlaylist = res2.text - expect(subPlaylist).to.contain(resolution + '_000.ts') + expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`) } } { - for (const resolution of resolutions) { - - const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`) + const baseUrl = 'http://localhost:9001/static/playlists/hls' - const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) - - const sha256Server = resSha.body[ resolution + '_000.ts' ] - expect(sha256(res2.body)).to.equal(sha256Server) + for (const resolution of resolutions) { + await checkSegmentHash(baseUrl, baseUrl, videoUUID, resolution, hlsPlaylist) } } } -- cgit v1.2.3 From 597a9266d426aa04c2f229168e4285a76bea2c12 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 7 Feb 2019 15:56:17 +0100 Subject: Add player mode in watch/embed urls --- server/lib/job-queue/handlers/video-file.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 7119ce0ca..04983155c 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -172,7 +172,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: Video // don't notify prior to scheduled video update if (!videoDatabase.ScheduleVideoUpdate) { - if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) + if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) } -- cgit v1.2.3 From 328c78bc4a570a9aceaaa1a2124bacd4a0e8d295 Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Sat, 6 Oct 2018 13:54:00 +0200 Subject: allow administration to change/reset a user's password --- server/controllers/api/users/index.ts | 1 + server/lib/emailer.ts | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'server') diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index dbe0718d4..beac6d8b1 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -3,6 +3,7 @@ import * as RateLimit from 'express-rate-limit' import { UserCreate, UserRight, UserRole, UserUpdate } from '../../../../shared' import { logger } from '../../../helpers/logger' import { getFormattedObjects } from '../../../helpers/utils' +import { pseudoRandomBytesPromise } from '../../../helpers/core-utils' import { CONFIG, RATES_LIMIT, sequelizeTypescript } from '../../../initializers' import { Emailer } from '../../../lib/emailer' import { Redis } from '../../../lib/redis' diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index f384a254e..7681164b3 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -101,6 +101,22 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } + addForceResetPasswordEmailJob (to: string, resetPasswordUrl: string) { + const text = `Hi dear user,\n\n` + + `Your password has been reset on ${CONFIG.WEBSERVER.HOST}! ` + + `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + + `Cheers,\n` + + `PeerTube.` + + const emailPayload: EmailPayload = { + to: [ to ], + subject: 'Reset of your PeerTube password', + text + } + + return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) + } + addNewFollowNotification (to: string[], actorFollow: ActorFollowModel, followType: 'account' | 'channel') { const followerName = actorFollow.ActorFollower.Account.getDisplayName() const followingName = (actorFollow.ActorFollowing.VideoChannel || actorFollow.ActorFollowing.Account).getDisplayName() -- cgit v1.2.3 From b426edd4854adc6e65844d8c54b8998e792b5778 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 11 Feb 2019 09:30:29 +0100 Subject: Cleanup reset user password by admin And add some tests --- server/controllers/api/users/index.ts | 20 ++++++++++---------- server/controllers/api/users/me.ts | 2 +- server/initializers/constants.ts | 2 ++ server/lib/emailer.ts | 20 ++------------------ server/middlewares/validators/users.ts | 2 ++ server/tests/api/check-params/users.ts | 18 ++++++++++++++++++ server/tests/api/users/users.ts | 16 ++++++++++++++++ 7 files changed, 51 insertions(+), 29 deletions(-) (limited to 'server') diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index beac6d8b1..e3533a7f6 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -3,7 +3,6 @@ import * as RateLimit from 'express-rate-limit' import { UserCreate, UserRight, UserRole, UserUpdate } from '../../../../shared' import { logger } from '../../../helpers/logger' import { getFormattedObjects } from '../../../helpers/utils' -import { pseudoRandomBytesPromise } from '../../../helpers/core-utils' import { CONFIG, RATES_LIMIT, sequelizeTypescript } from '../../../initializers' import { Emailer } from '../../../lib/emailer' import { Redis } from '../../../lib/redis' @@ -230,7 +229,7 @@ async function unblockUser (req: express.Request, res: express.Response, next: e return res.status(204).end() } -async function blockUser (req: express.Request, res: express.Response, next: express.NextFunction) { +async function blockUser (req: express.Request, res: express.Response) { const user: UserModel = res.locals.user const reason = req.body.reason @@ -239,23 +238,23 @@ async function blockUser (req: express.Request, res: express.Response, next: exp return res.status(204).end() } -function getUser (req: express.Request, res: express.Response, next: express.NextFunction) { +function getUser (req: express.Request, res: express.Response) { return res.json((res.locals.user as UserModel).toFormattedJSON()) } -async function autocompleteUsers (req: express.Request, res: express.Response, next: express.NextFunction) { +async function autocompleteUsers (req: express.Request, res: express.Response) { const resultList = await UserModel.autoComplete(req.query.search as string) return res.json(resultList) } -async function listUsers (req: express.Request, res: express.Response, next: express.NextFunction) { +async function listUsers (req: express.Request, res: express.Response) { const resultList = await UserModel.listForApi(req.query.start, req.query.count, req.query.sort, req.query.search) return res.json(getFormattedObjects(resultList.data, resultList.total)) } -async function removeUser (req: express.Request, res: express.Response, next: express.NextFunction) { +async function removeUser (req: express.Request, res: express.Response) { const user: UserModel = res.locals.user await user.destroy() @@ -265,12 +264,13 @@ async function removeUser (req: express.Request, res: express.Response, next: ex return res.sendStatus(204) } -async function updateUser (req: express.Request, res: express.Response, next: express.NextFunction) { +async function updateUser (req: express.Request, res: express.Response) { const body: UserUpdate = req.body const userToUpdate = res.locals.user as UserModel const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON()) const roleChanged = body.role !== undefined && body.role !== userToUpdate.role + if (body.password !== undefined) userToUpdate.password = body.password if (body.email !== undefined) userToUpdate.email = body.email if (body.emailVerified !== undefined) userToUpdate.emailVerified = body.emailVerified if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota @@ -280,11 +280,11 @@ async function updateUser (req: express.Request, res: express.Response, next: ex const user = await userToUpdate.save() // Destroy user token to refresh rights - if (roleChanged) await deleteUserToken(userToUpdate.id) + if (roleChanged || body.password !== undefined) await deleteUserToken(userToUpdate.id) auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) - // Don't need to send this update to followers, these attributes are not propagated + // Don't need to send this update to followers, these attributes are not federated return res.sendStatus(204) } @@ -294,7 +294,7 @@ async function askResetUserPassword (req: express.Request, res: express.Response const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) const url = CONFIG.WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString - await Emailer.Instance.addForgetPasswordEmailJob(user.email, url) + await Emailer.Instance.addPasswordResetEmailJob(user.email, url) return res.status(204).end() } diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 94a2b8732..d5e154869 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -167,7 +167,7 @@ async function deleteMe (req: express.Request, res: express.Response) { return res.sendStatus(204) } -async function updateMe (req: express.Request, res: express.Response, next: express.NextFunction) { +async function updateMe (req: express.Request, res: express.Response) { const body: UserUpdateMe = req.body const user: UserModel = res.locals.oauth.token.user diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 98f8f8694..e5c4c4e63 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -711,6 +711,8 @@ if (isTestInstance() === true) { CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' + + RATES_LIMIT.LOGIN.MAX = 20 } updateWebserverUrls() diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 7681164b3..672414cc0 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -101,22 +101,6 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - addForceResetPasswordEmailJob (to: string, resetPasswordUrl: string) { - const text = `Hi dear user,\n\n` + - `Your password has been reset on ${CONFIG.WEBSERVER.HOST}! ` + - `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + - `Cheers,\n` + - `PeerTube.` - - const emailPayload: EmailPayload = { - to: [ to ], - subject: 'Reset of your PeerTube password', - text - } - - return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) - } - addNewFollowNotification (to: string[], actorFollow: ActorFollowModel, followType: 'account' | 'channel') { const followerName = actorFollow.ActorFollower.Account.getDisplayName() const followingName = (actorFollow.ActorFollowing.VideoChannel || actorFollow.ActorFollowing.Account).getDisplayName() @@ -312,9 +296,9 @@ class Emailer { return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) } - addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { + addPasswordResetEmailJob (to: string, resetPasswordUrl: string) { const text = `Hi dear user,\n\n` + - `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + + `A reset password procedure for your account ${to} has been requested on ${CONFIG.WEBSERVER.HOST} ` + `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + `If you are not the person who initiated this request, please ignore this email.\n\n` + `Cheers,\n` + diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 1bb0bfb1b..a52e3060a 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -113,6 +113,7 @@ const deleteMeValidator = [ const usersUpdateValidator = [ param('id').isInt().not().isEmpty().withMessage('Should have a valid id'), + body('password').optional().custom(isUserPasswordValid).withMessage('Should have a valid password'), body('email').optional().isEmail().withMessage('Should have a valid email attribute'), body('emailVerified').optional().isBoolean().withMessage('Should have a valid email verified attribute'), body('videoQuota').optional().custom(isUserVideoQuotaValid).withMessage('Should have a valid user quota'), @@ -233,6 +234,7 @@ const usersAskResetPasswordValidator = [ logger.debug('Checking usersAskResetPassword parameters', { parameters: req.body }) if (areValidationErrors(req, res)) return + const exists = await checkUserEmailExist(req.body.email, res, false) if (!exists) { logger.debug('User with email %s does not exist (asking reset password).', req.body.email) diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index a3e8e2e9c..13be8b460 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -464,6 +464,24 @@ describe('Test users API validators', function () { await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) }) + it('Should fail with a too small password', async function () { + const fields = { + currentPassword: 'my super password', + password: 'bla' + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + + it('Should fail with a too long password', async function () { + const fields = { + currentPassword: 'my super password', + password: 'super'.repeat(61) + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + it('Should fail with an non authenticated user', async function () { const fields = { videoQuota: 42 diff --git a/server/tests/api/users/users.ts b/server/tests/api/users/users.ts index ad98ab1c7..c4465d541 100644 --- a/server/tests/api/users/users.ts +++ b/server/tests/api/users/users.ts @@ -501,6 +501,22 @@ describe('Test users', function () { accessTokenUser = await userLogin(server, user) }) + it('Should be able to update another user password', async function () { + await updateUser({ + url: server.url, + userId, + accessToken, + password: 'password updated' + }) + + await getMyUserVideoQuotaUsed(server.url, accessTokenUser, 401) + + await userLogin(server, user, 400) + + user.password = 'password updated' + accessTokenUser = await userLogin(server, user) + }) + it('Should be able to list video blacklist by a moderator', async function () { await getBlacklistedVideosList(server.url, accessTokenUser) }) -- cgit v1.2.3 From e79d0ba56c4d5518c6ccab74e805c449a43e1c76 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 11 Feb 2019 11:01:50 +0100 Subject: Fix reverse proxy test --- server/tests/api/server/reverse-proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/tests/api/server/reverse-proxy.ts b/server/tests/api/server/reverse-proxy.ts index d4c08c346..ee0fffd5a 100644 --- a/server/tests/api/server/reverse-proxy.ts +++ b/server/tests/api/server/reverse-proxy.ts @@ -95,7 +95,7 @@ describe('Test application behind a reverse proxy', function () { it('Should rate limit logins', async function () { const user = { username: 'root', password: 'fail' } - for (let i = 0; i < 14; i++) { + for (let i = 0; i < 19; i++) { await userLogin(server, user, 400) } -- cgit v1.2.3 From 53a94c7cfa8368da4cd248d65df8346905938f0c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 11 Feb 2019 11:48:56 +0100 Subject: Add federation tests on download enabled --- server/tests/api/videos/multiple-servers.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'server') diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 256be5d1c..99b74ccff 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -918,11 +918,12 @@ describe('Test multiple servers', function () { } }) - it('Should disable comments', async function () { + it('Should disable comments and download', async function () { this.timeout(20000) const attributes = { - commentsEnabled: false + commentsEnabled: false, + downloadEnabled: false } await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, attributes) @@ -932,6 +933,7 @@ describe('Test multiple servers', function () { for (const server of servers) { const res = await getVideo(server.url, videoUUID) expect(res.body.commentsEnabled).to.be.false + expect(res.body.downloadEnabled).to.be.false const text = 'my super forbidden comment' await addVideoCommentThread(server.url, server.accessToken, videoUUID, text, 409) -- cgit v1.2.3 From 25451e08c71b81ee3da75d65eab22445a78dd0c2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 11 Feb 2019 11:55:11 +0100 Subject: Update migration version for download enabled --- server/initializers/constants.ts | 2 +- .../migrations/0280-video-downloading-enabled.ts | 27 ---------------------- .../migrations/0335-video-downloading-enabled.ts | 27 ++++++++++++++++++++++ 3 files changed, 28 insertions(+), 28 deletions(-) delete mode 100644 server/initializers/migrations/0280-video-downloading-enabled.ts create mode 100644 server/initializers/migrations/0335-video-downloading-enabled.ts (limited to 'server') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e5c4c4e63..639493d73 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -16,7 +16,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 330 +const LAST_MIGRATION_VERSION = 335 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0280-video-downloading-enabled.ts b/server/initializers/migrations/0280-video-downloading-enabled.ts deleted file mode 100644 index e79466447..000000000 --- a/server/initializers/migrations/0280-video-downloading-enabled.ts +++ /dev/null @@ -1,27 +0,0 @@ -import * as Sequelize from 'sequelize' -import { Migration } from '../../models/migrations' - -async function up (utils: { - transaction: Sequelize.Transaction, - queryInterface: Sequelize.QueryInterface, - sequelize: Sequelize.Sequelize -}): Promise { - const data = { - type: Sequelize.BOOLEAN, - allowNull: false, - defaultValue: true - } as Migration.Boolean - await utils.queryInterface.addColumn('video', 'downloadEnabled', data) - - data.defaultValue = null - return utils.queryInterface.changeColumn('video', 'downloadEnabled', data) -} - -function down (options) { - throw new Error('Not implemented.') -} - -export { - up, - down -} diff --git a/server/initializers/migrations/0335-video-downloading-enabled.ts b/server/initializers/migrations/0335-video-downloading-enabled.ts new file mode 100644 index 000000000..e79466447 --- /dev/null +++ b/server/initializers/migrations/0335-video-downloading-enabled.ts @@ -0,0 +1,27 @@ +import * as Sequelize from 'sequelize' +import { Migration } from '../../models/migrations' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize +}): Promise { + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: true + } as Migration.Boolean + await utils.queryInterface.addColumn('video', 'downloadEnabled', data) + + data.defaultValue = null + return utils.queryInterface.changeColumn('video', 'downloadEnabled', data) +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} -- cgit v1.2.3