matrix:
include:
- env: TEST_SUITE=misc
- - env: TEST_SUITE=api-fast
- - env: TEST_SUITE=api-slow
+ - env: TEST_SUITE=api-1
+ - env: TEST_SUITE=api-2
+ - env: TEST_SUITE=api-3
- env: TEST_SUITE=cli
- env: TEST_SUITE=lint
log:
level: 'info' # debug/info/warning/error
+search:
+ remote_uri: # Add ability to fetch remote videos/actors by their URI, that may not be federated with your instance
+ users: true
+ anonymous: false
+
cache:
previews:
size: 500 # Max number of previews you want to cache
log:
level: 'info' # debug/info/warning/error
+search:
+ remote_uri: # Add ability to search remote videos/actors by URI, that may not be federated with your instance
+ users: true
+ anonymous: false
###############################################################################
#
if [ "$1" = "misc" ]; then
npm run build
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts \
- server/tests/feeds/feeds.ts
+ server/tests/feeds/index.ts
elif [ "$1" = "api" ]; then
npm run build:server
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index.ts
elif [ "$1" = "cli" ]; then
npm run build:server
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/cli/index.ts
-elif [ "$1" = "api-fast" ]; then
+elif [ "$1" = "api-1" ]; then
npm run build:server
- mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-fast.ts
-elif [ "$1" = "api-slow" ]; then
+ mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-1.ts
+elif [ "$1" = "api-2" ]; then
npm run build:server
- mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-slow.ts
+ mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-2.ts
+elif [ "$1" = "api-3" ]; then
+ npm run build:server
+ mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts
elif [ "$1" = "lint" ]; then
( cd client
npm run lint
videosSearchSortValidator
} from '../../middlewares'
import { VideosSearchQuery } from '../../../shared/models/search'
-import { getOrCreateAccountAndVideoAndChannel } from '../../lib/activitypub'
+import { getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
import { logger } from '../../helpers/logger'
+import { User } from '../../../shared/models/users'
+import { CONFIG } from '../../initializers/constants'
const searchRouter = express.Router()
async function searchVideoUrl (url: string, res: express.Response) {
let video: VideoModel
+ const user: User = res.locals.oauth ? res.locals.oauth.token.User : undefined
- try {
- const syncParam = {
- likes: false,
- dislikes: false,
- shares: false,
- comments: false,
- thumbnail: true
- }
+ // Check if we can fetch a remote video with the URL
+ if (
+ CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
+ (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
+ ) {
+ try {
+ const syncParam = {
+ likes: false,
+ dislikes: false,
+ shares: false,
+ comments: false,
+ thumbnail: true,
+ refreshVideo: false
+ }
- const res = await getOrCreateAccountAndVideoAndChannel(url, syncParam)
- video = res ? res.video : undefined
- } catch (err) {
- logger.info('Cannot search remote video %s.', url)
+ const res = await getOrCreateVideoAndAccountAndChannel(url, syncParam)
+ video = res ? res.video : undefined
+ } catch (err) {
+ logger.info('Cannot search remote video %s.', url)
+ }
+ } else {
+ video = await VideoModel.loadByUrlAndPopulateAccount(url)
}
return res.json({
LOG: {
LEVEL: config.get<string>('log.level')
},
+ SEARCH: {
+ REMOTE_URI: {
+ USERS: config.get<boolean>('search.remote_uri.users'),
+ ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous')
+ }
+ },
ADMIN: {
get EMAIL () { return config.get<string>('admin.email') }
},
MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
},
MAX_RECURSION_COMMENTS: 100,
- ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day
+ ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000, // 1 day
+ VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day
}
const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
+ ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
import { VideoShareModel } from '../../../models/video/video-share'
import { getOrCreateActorAndServerAndModel } from '../actor'
import { forwardVideoRelatedActivity } from '../send/utils'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
async function processAnnounceActivity (activity: ActivityAnnounce) {
const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
- const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri)
return sequelizeTypescript.transaction(async t => {
// Add share entry
import { VideoCommentModel } from '../../../models/video/video-comment'
import { getOrCreateActorAndServerAndModel } from '../actor'
import { resolveThread } from '../video-comments'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
async function processCreateActivity (activity: ActivityCreate) {
async function processCreateVideo (activity: ActivityCreate) {
const videoToCreateData = activity.object as VideoTorrentObject
- const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData)
return video
}
if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
- const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
return sequelizeTypescript.transaction(async t => {
const rate = {
async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
const view = activity.object as ViewObject
- const { video } = await getOrCreateAccountAndVideoAndChannel(view.object)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(view.object)
const actor = await ActorModel.loadByUrl(view.actor)
if (!actor) throw new Error('Unknown actor ' + view.actor)
const account = actor.Account
if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
- const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object)
return sequelizeTypescript.transaction(async t => {
const videoAbuseData = {
import { ActorModel } from '../../../models/activitypub/actor'
import { getOrCreateActorAndServerAndModel } from '../actor'
import { forwardVideoRelatedActivity } from '../send/utils'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
async function processLikeActivity (activity: ActivityLike) {
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
const byAccount = byActor.Account
if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
- const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl)
return sequelizeTypescript.transaction(async t => {
const rate = {
import { ActorModel } from '../../../models/activitypub/actor'
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { forwardVideoRelatedActivity } from '../send/utils'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
import { VideoShareModel } from '../../../models/video/video-share'
async function processUndoActivity (activity: ActivityUndo) {
async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
const likeActivity = activity.object as ActivityLike
- const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object)
return sequelizeTypescript.transaction(async t => {
const byAccount = await AccountModel.loadByUrl(actorUrl, t)
async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
const dislike = activity.object.object as DislikeObject
- const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
return sequelizeTypescript.transaction(async t => {
const byAccount = await AccountModel.loadByUrl(actorUrl, t)
-import * as Bluebird from 'bluebird'
import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
import { sequelizeTypescript } from '../../../initializers'
import { AccountModel } from '../../../models/account/account'
import { ActorModel } from '../../../models/activitypub/actor'
-import { TagModel } from '../../../models/video/tag'
import { VideoChannelModel } from '../../../models/video/video-channel'
-import { VideoFileModel } from '../../../models/video/video-file'
import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
-import {
- generateThumbnailFromUrl,
- getOrCreateAccountAndVideoAndChannel,
- getOrCreateVideoChannel,
- videoActivityObjectToDBAttributes,
- videoFileActivityUrlToDBAttributes
-} from '../videos'
+import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannel, updateVideoFromAP } from '../videos'
import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
-import { VideoCaptionModel } from '../../../models/video/video-caption'
async function processUpdateActivity (activity: ActivityUpdate) {
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
return undefined
}
- const res = await getOrCreateAccountAndVideoAndChannel(videoObject.id)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
+ const channelActor = await getOrCreateVideoChannel(videoObject)
- // Fetch video channel outside the transaction
- const newVideoChannelActor = await getOrCreateVideoChannel(videoObject)
- const newVideoChannel = newVideoChannelActor.VideoChannel
-
- logger.debug('Updating remote video "%s".', videoObject.uuid)
- let videoInstance = res.video
- let videoFieldsSave: any
-
- try {
- await sequelizeTypescript.transaction(async t => {
- const sequelizeOptions = {
- transaction: t
- }
-
- videoFieldsSave = videoInstance.toJSON()
-
- // Check actor has the right to update the video
- const videoChannel = videoInstance.VideoChannel
- if (videoChannel.Account.Actor.id !== actor.id) {
- throw new Error('Account ' + actor.url + ' does not own video channel ' + videoChannel.Actor.url)
- }
-
- const videoData = await videoActivityObjectToDBAttributes(newVideoChannel, videoObject, activity.to)
- videoInstance.set('name', videoData.name)
- videoInstance.set('uuid', videoData.uuid)
- videoInstance.set('url', videoData.url)
- videoInstance.set('category', videoData.category)
- videoInstance.set('licence', videoData.licence)
- videoInstance.set('language', videoData.language)
- videoInstance.set('description', videoData.description)
- videoInstance.set('support', videoData.support)
- videoInstance.set('nsfw', videoData.nsfw)
- videoInstance.set('commentsEnabled', videoData.commentsEnabled)
- videoInstance.set('waitTranscoding', videoData.waitTranscoding)
- videoInstance.set('state', videoData.state)
- videoInstance.set('duration', videoData.duration)
- videoInstance.set('createdAt', videoData.createdAt)
- videoInstance.set('updatedAt', videoData.updatedAt)
- videoInstance.set('views', videoData.views)
- videoInstance.set('privacy', videoData.privacy)
- videoInstance.set('channelId', videoData.channelId)
-
- await videoInstance.save(sequelizeOptions)
-
- // Don't block on request
- generateThumbnailFromUrl(videoInstance, videoObject.icon)
- .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
-
- // Remove old video files
- const videoFileDestroyTasks: Bluebird<void>[] = []
- for (const videoFile of videoInstance.VideoFiles) {
- videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
- }
- await Promise.all(videoFileDestroyTasks)
-
- const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject)
- const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
- await Promise.all(tasks)
-
- // Update Tags
- const tags = videoObject.tag.map(tag => tag.name)
- const tagInstances = await TagModel.findOrCreateTags(tags, t)
- await videoInstance.$set('Tags', tagInstances, sequelizeOptions)
-
- // Update captions
- await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoInstance.id, t)
-
- const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
- return VideoCaptionModel.insertOrReplaceLanguage(videoInstance.id, c.identifier, t)
- })
- await Promise.all(videoCaptionsPromises)
- })
-
- logger.info('Remote video with uuid %s updated', videoObject.uuid)
- } catch (err) {
- if (videoInstance !== undefined && videoFieldsSave !== undefined) {
- resetSequelizeInstance(videoInstance, videoFieldsSave)
- }
-
- // This is just a debug because we will retry the insert
- logger.debug('Cannot update the remote video.', { err })
- throw err
- }
+ return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to)
}
async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
import { sendUndoAnnounce, sendVideoAnnounce } from './send'
import { getAnnounceActivityPubUrl } from './url'
import { VideoChannelModel } from '../../models/video/video-channel'
+import * as Bluebird from 'bluebird'
+import { doRequest } from '../../helpers/requests'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { logger } from '../../helpers/logger'
+import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
if (video.privacy === VideoPrivacy.PRIVATE) return undefined
await shareByVideoChannel(video, t)
}
+async function addVideoShares (shareUrls: string[], instance: VideoModel) {
+ await Bluebird.map(shareUrls, async shareUrl => {
+ try {
+ // Fetch url
+ const { body } = await doRequest({
+ uri: shareUrl,
+ json: true,
+ activityPub: true
+ })
+ if (!body || !body.actor) throw new Error('Body of body actor is invalid')
+
+ const actorUrl = body.actor
+ const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+
+ const entry = {
+ actorId: actor.id,
+ videoId: instance.id,
+ url: shareUrl
+ }
+
+ await VideoShareModel.findOrCreate({
+ where: {
+ url: shareUrl
+ },
+ defaults: entry
+ })
+ } catch (err) {
+ logger.warn('Cannot add share %s.', shareUrl, { err })
+ }
+ }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+}
+
export {
changeVideoChannelShare,
+ addVideoShares,
shareVideoByServerAndChannel
}
import { VideoModel } from '../../models/video/video'
import { VideoCommentModel } from '../../models/video/video-comment'
import { getOrCreateActorAndServerAndModel } from './actor'
-import { getOrCreateAccountAndVideoAndChannel } from './videos'
+import { getOrCreateVideoAndAccountAndChannel } from './videos'
import * as Bluebird from 'bluebird'
async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
try {
// Maybe it's a reply to a video?
- const { video } = await getOrCreateAccountAndVideoAndChannel(url)
+ const { video } = await getOrCreateVideoAndAccountAndChannel(url)
if (comments.length !== 0) {
const firstReply = comments[ comments.length - 1 ]
import { AccountModel } from '../../models/account/account'
import { VideoModel } from '../../models/video/video'
import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send'
+import { VideoRateType } from '../../../shared/models/videos'
+import * as Bluebird from 'bluebird'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { AccountVideoRateModel } from '../../models/account/account-video-rate'
+import { logger } from '../../helpers/logger'
+import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
+
+async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
+ let rateCounts = 0
+
+ await Bluebird.map(actorUrls, async actorUrl => {
+ try {
+ const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+ const [ , created ] = await AccountVideoRateModel
+ .findOrCreate({
+ where: {
+ videoId: video.id,
+ accountId: actor.Account.id
+ },
+ defaults: {
+ videoId: video.id,
+ accountId: actor.Account.id,
+ type: rate
+ }
+ })
+
+ if (created) rateCounts += 1
+ } catch (err) {
+ logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
+ }
+ }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+
+ logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
+
+ // This is "likes" and "dislikes"
+ if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts })
+
+ return
+}
async function sendVideoRateChange (account: AccountModel,
video: VideoModel,
}
export {
+ createRates,
sendVideoRateChange
}
import * as request from 'request'
import { ActivityIconObject, VideoState } from '../../../shared/index'
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
-import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
+import { VideoPrivacy } from '../../../shared/models/videos'
import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
-import { retryTransactionWrapper } from '../../helpers/database-utils'
+import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
-import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
-import { AccountVideoRateModel } from '../../models/account/account-video-rate'
+import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
import { ActorModel } from '../../models/activitypub/actor'
import { TagModel } from '../../models/video/tag'
import { VideoModel } from '../../models/video/video'
import { VideoChannelModel } from '../../models/video/video-channel'
import { VideoFileModel } from '../../models/video/video-file'
-import { VideoShareModel } from '../../models/video/video-share'
-import { getOrCreateActorAndServerAndModel } from './actor'
+import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
import { addVideoComments } from './video-comments'
import { crawlCollectionPage } from './crawl'
import { sendCreateVideo, sendUpdateVideo } from './send'
-import { shareVideoByServerAndChannel } from './index'
import { isArray } from '../../helpers/custom-validators/misc'
import { VideoCaptionModel } from '../../models/video/video-caption'
import { JobQueue } from '../job-queue'
import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
+import { getUrlFromWebfinger } from '../../helpers/webfinger'
+import { createRates } from './video-rates'
+import { addVideoShares, shareVideoByServerAndChannel } from './share'
+import { AccountModel } from '../../models/account/account'
async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
// If the video is not private and published, we federate it
return getOrCreateActorAndServerAndModel(channel.id)
}
-async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
+async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
logger.debug('Adding remote video %s.', videoObject.id)
const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
- const sequelizeOptions = {
- transaction: t
- }
- const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t)
- if (videoFromDatabase) return videoFromDatabase
+ const sequelizeOptions = { transaction: t }
const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
const video = VideoModel.build(videoData)
}
type SyncParam = {
- likes: boolean,
- dislikes: boolean,
- shares: boolean,
- comments: boolean,
+ likes: boolean
+ dislikes: boolean
+ shares: boolean
+ comments: boolean
thumbnail: boolean
+ refreshVideo: boolean
}
-async function getOrCreateAccountAndVideoAndChannel (
+async function getOrCreateVideoAndAccountAndChannel (
videoObject: VideoTorrentObject | string,
- syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true }
+ syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
) {
const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
- const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
- if (videoFromDatabase) return { video: videoFromDatabase }
+ let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
+ if (videoFromDatabase) {
+ const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
+ if (syncParam.refreshVideo === true) videoFromDatabase = await p
+
+ return { video: videoFromDatabase }
+ }
- const fetchedVideo = await fetchRemoteVideo(videoUrl)
+ const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
const channelActor = await getOrCreateVideoChannel(fetchedVideo)
- const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail)
+ const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
// Process outside the transaction because we could fetch remote data
return { video }
}
-async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
- let rateCounts = 0
-
- await Bluebird.map(actorUrls, async actorUrl => {
- try {
- const actor = await getOrCreateActorAndServerAndModel(actorUrl)
- const [ , created ] = await AccountVideoRateModel
- .findOrCreate({
- where: {
- videoId: video.id,
- accountId: actor.Account.id
- },
- defaults: {
- videoId: video.id,
- accountId: actor.Account.id,
- type: rate
- }
- })
-
- if (created) rateCounts += 1
- } catch (err) {
- logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
+async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
+ const options = {
+ uri: videoUrl,
+ method: 'GET',
+ json: true,
+ activityPub: true
+ }
+
+ logger.info('Fetching remote video %s.', videoUrl)
+
+ const { response, body } = await doRequest(options)
+
+ if (sanitizeAndCheckVideoTorrentObject(body) === false) {
+ logger.debug('Remote video JSON is not valid.', { body })
+ return { response, videoObject: undefined }
+ }
+
+ return { response, videoObject: body }
+}
+
+async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
+ if (!video.isOutdated()) return video
+
+ try {
+ const { response, videoObject } = await fetchRemoteVideo(video.url)
+ if (response.statusCode === 404) {
+ // Video does not exist anymore
+ await video.destroy()
+ return undefined
}
- }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
- logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
+ if (videoObject === undefined) {
+ logger.warn('Cannot refresh remote video: invalid body.')
+ return video
+ }
- // This is "likes" and "dislikes"
- if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts })
+ const channelActor = await getOrCreateVideoChannel(videoObject)
+ const account = await AccountModel.load(channelActor.VideoChannel.accountId)
+ return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
- return
+ } catch (err) {
+ logger.warn('Cannot refresh video.', { err })
+ return video
+ }
}
-async function addVideoShares (shareUrls: string[], instance: VideoModel) {
- await Bluebird.map(shareUrls, async shareUrl => {
- try {
- // Fetch url
- const { body } = await doRequest({
- uri: shareUrl,
- json: true,
- activityPub: true
- })
- if (!body || !body.actor) throw new Error('Body of body actor is invalid')
+async function updateVideoFromAP (
+ video: VideoModel,
+ videoObject: VideoTorrentObject,
+ accountActor: ActorModel,
+ channelActor: ActorModel,
+ overrideTo?: string[]
+) {
+ logger.debug('Updating remote video "%s".', videoObject.uuid)
+ let videoFieldsSave: any
+
+ try {
+ const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
+ const sequelizeOptions = {
+ transaction: t
+ }
- const actorUrl = body.actor
- const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+ videoFieldsSave = video.toJSON()
- const entry = {
- actorId: actor.id,
- videoId: instance.id,
- url: shareUrl
+ // Check actor has the right to update the video
+ const videoChannel = video.VideoChannel
+ if (videoChannel.Account.Actor.id !== accountActor.id) {
+ throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
}
- await VideoShareModel.findOrCreate({
- where: {
- url: shareUrl
- },
- defaults: entry
- })
- } catch (err) {
- logger.warn('Cannot add share %s.', shareUrl, { err })
- }
- }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
-}
+ const to = overrideTo ? overrideTo : videoObject.to
+ const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
+ video.set('name', videoData.name)
+ video.set('uuid', videoData.uuid)
+ video.set('url', videoData.url)
+ video.set('category', videoData.category)
+ video.set('licence', videoData.licence)
+ video.set('language', videoData.language)
+ video.set('description', videoData.description)
+ video.set('support', videoData.support)
+ video.set('nsfw', videoData.nsfw)
+ video.set('commentsEnabled', videoData.commentsEnabled)
+ video.set('waitTranscoding', videoData.waitTranscoding)
+ video.set('state', videoData.state)
+ video.set('duration', videoData.duration)
+ video.set('createdAt', videoData.createdAt)
+ video.set('publishedAt', videoData.publishedAt)
+ video.set('views', videoData.views)
+ video.set('privacy', videoData.privacy)
+ video.set('channelId', videoData.channelId)
+
+ await video.save(sequelizeOptions)
+
+ // Don't block on request
+ generateThumbnailFromUrl(video, videoObject.icon)
+ .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
+
+ // Remove old video files
+ const videoFileDestroyTasks: Bluebird<void>[] = []
+ for (const videoFile of video.VideoFiles) {
+ videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
+ }
+ await Promise.all(videoFileDestroyTasks)
-async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> {
- const options = {
- uri: videoUrl,
- method: 'GET',
- json: true,
- activityPub: true
- }
+ const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
+ const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
+ await Promise.all(tasks)
- logger.info('Fetching remote video %s.', videoUrl)
+ // Update Tags
+ const tags = videoObject.tag.map(tag => tag.name)
+ const tagInstances = await TagModel.findOrCreateTags(tags, t)
+ await video.$set('Tags', tagInstances, sequelizeOptions)
- const { body } = await doRequest(options)
+ // Update captions
+ await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
- if (sanitizeAndCheckVideoTorrentObject(body) === false) {
- logger.debug('Remote video JSON is not valid.', { body })
- return undefined
- }
+ const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
+ return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
+ })
+ await Promise.all(videoCaptionsPromises)
+ })
+
+ logger.info('Remote video with uuid %s updated', videoObject.uuid)
- return body
+ return updatedVideo
+ } catch (err) {
+ if (video !== undefined && videoFieldsSave !== undefined) {
+ resetSequelizeInstance(video, videoFieldsSave)
+ }
+
+ // This is just a debug because we will retry the insert
+ logger.debug('Cannot update the remote video.', { err })
+ throw err
+ }
}
export {
+ updateVideoFromAP,
federateVideoIfNeeded,
fetchRemoteVideo,
- getOrCreateAccountAndVideoAndChannel,
+ getOrCreateVideoAndAccountAndChannel,
fetchRemoteVideoStaticFile,
fetchRemoteVideoDescription,
generateThumbnailFromUrl,
videoActivityObjectToDBAttributes,
videoFileActivityUrlToDBAttributes,
- getOrCreateVideo,
+ createVideo,
getOrCreateVideoChannel,
addVideoShares,
createRates
import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import {
+ ACTIVITY_PUB,
API_VERSION,
CONFIG,
CONSTRAINTS_FIELDS,
return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
}
- static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) {
- const query: IFindOptions<VideoModel> = {
- where: {
- [Sequelize.Op.or]: [
- { uuid },
- { url }
- ]
- }
- }
-
- if (t !== undefined) query.transaction = t
-
- return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
- }
-
static loadAndPopulateAccountAndServerAndTags (id: number) {
const options = {
order: [ [ 'Tags', 'name', 'ASC' ] ]
return 'PT' + this.duration + 'S'
}
+ isOutdated () {
+ if (this.isOwned()) return false
+
+ const now = Date.now()
+ const createdAtTime = this.createdAt.getTime()
+ const updatedAtTime = this.updatedAt.getTime()
+
+ return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL &&
+ (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
+ }
+
private getBaseUrls () {
let baseUrlHttp
let baseUrlWs
--- /dev/null
+import './check-params'
+import './search'
--- /dev/null
+import './server'
+import './users'
--- /dev/null
+import './videos'
+++ /dev/null
-// Order of the tests we want to execute
-import './server/stats'
-import './check-params'
-import './users/users'
-import './videos/single-server'
-import './videos/video-abuse'
-import './videos/video-captions'
-import './videos/video-blacklist'
-import './videos/video-blacklist-management'
-import './videos/video-description'
-import './videos/video-nsfw'
-import './videos/video-privacy'
-import './videos/services'
-import './server/email'
-import './server/config'
-import './server/reverse-proxy'
-import './search/search-videos'
-import './server/tracker'
+++ /dev/null
-// Order of the tests we want to execute
-import './videos/video-channels'
-import './videos/video-transcoder'
-import './videos/multiple-servers'
-import './server/follows'
-import './server/jobs'
-import './videos/video-comments'
-import './users/users-multiple-servers'
-import './users/user-subscriptions'
-import './server/handle-down'
-import './videos/video-schedule-update'
-import './videos/video-imports'
// Order of the tests we want to execute
-import './index-fast'
-import './index-slow'
+import './index-1'
+import './index-2'
+import './index-3'
--- /dev/null
+import './search-activitypub-videos'
+import './search-videos'
--- /dev/null
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import {
+ addVideoChannel,
+ flushAndRunMultipleServers,
+ flushTests,
+ getVideosList,
+ killallServers,
+ removeVideo,
+ searchVideoWithToken,
+ ServerInfo,
+ setAccessTokensToServers,
+ updateVideo,
+ uploadVideo,
+ wait,
+ searchVideo
+} from '../../utils'
+import { waitJobs } from '../../utils/server/jobs'
+import { Video, VideoPrivacy } from '../../../../shared/models/videos'
+
+const expect = chai.expect
+
+describe('Test a ActivityPub videos search', function () {
+ let servers: ServerInfo[]
+ let videoServer1UUID: string
+ let videoServer2UUID: string
+
+ before(async function () {
+ this.timeout(120000)
+
+ await flushTests()
+
+ servers = await flushAndRunMultipleServers(2)
+
+ await setAccessTokensToServers(servers)
+
+ {
+ const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1 on server 1' })
+ videoServer1UUID = res.body.video.uuid
+ }
+
+ {
+ const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 on server 2' })
+ videoServer2UUID = res.body.video.uuid
+ }
+
+ await waitJobs(servers)
+ })
+
+ it('Should not find a remote video', async function () {
+ {
+ const res = await searchVideoWithToken(servers[ 0 ].url, 'http://localhost:9002/videos/watch/43', servers[ 0 ].accessToken)
+
+ expect(res.body.total).to.equal(0)
+ expect(res.body.data).to.be.an('array')
+ expect(res.body.data).to.have.lengthOf(0)
+ }
+
+ {
+ const res = await searchVideo(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID)
+
+ expect(res.body.total).to.equal(0)
+ expect(res.body.data).to.be.an('array')
+ expect(res.body.data).to.have.lengthOf(0)
+ }
+ })
+
+ it('Should search a local video', async function () {
+ const res = await searchVideo(servers[0].url, 'http://localhost:9001/videos/watch/' + videoServer1UUID)
+
+ expect(res.body.total).to.equal(1)
+ expect(res.body.data).to.be.an('array')
+ expect(res.body.data).to.have.lengthOf(1)
+ expect(res.body.data[0].name).to.equal('video 1 on server 1')
+ })
+
+ it('Should search a remote video', async function () {
+ const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+
+ expect(res.body.total).to.equal(1)
+ expect(res.body.data).to.be.an('array')
+ expect(res.body.data).to.have.lengthOf(1)
+ expect(res.body.data[0].name).to.equal('video 1 on server 2')
+ })
+
+ it('Should not list this remote video', async function () {
+ const res = await getVideosList(servers[0].url)
+ expect(res.body.total).to.equal(1)
+ expect(res.body.data).to.have.lengthOf(1)
+ expect(res.body.data[0].name).to.equal('video 1 on server 1')
+ })
+
+ it('Should update video of server 2, and refresh it on server 1', async function () {
+ this.timeout(60000)
+
+ const channelAttributes = {
+ name: 'super_channel',
+ displayName: 'super channel'
+ }
+ const resChannel = await addVideoChannel(servers[1].url, servers[1].accessToken, channelAttributes)
+ const videoChannelId = resChannel.body.videoChannel.id
+
+ const attributes = {
+ name: 'updated',
+ tag: [ 'tag1', 'tag2' ],
+ privacy: VideoPrivacy.UNLISTED,
+ channelId: videoChannelId
+ }
+ await updateVideo(servers[1].url, servers[1].accessToken, videoServer2UUID, attributes)
+
+ await waitJobs(servers)
+ // Expire video
+ await wait(10000)
+
+ // Will run refresh async
+ await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+
+ // Wait refresh
+ await wait(5000)
+
+ const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+ expect(res.body.total).to.equal(1)
+ expect(res.body.data).to.have.lengthOf(1)
+
+ const video: Video = res.body.data[0]
+ expect(video.name).to.equal('updated')
+ expect(video.channel.name).to.equal('super_channel')
+ expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
+ })
+
+ it('Should delete video of server 2, and delete it on server 1', async function () {
+ this.timeout(60000)
+
+ await removeVideo(servers[1].url, servers[1].accessToken, videoServer2UUID)
+
+ await waitJobs(servers)
+ // Expire video
+ await wait(10000)
+
+ // Will run refresh async
+ await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+
+ // Wait refresh
+ await wait(5000)
+
+ const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+ expect(res.body.total).to.equal(0)
+ expect(res.body.data).to.have.lengthOf(0)
+ })
+
+ after(async function () {
+ killallServers(servers)
+
+ // Keep the logs if the test failed
+ if (this['ok']) {
+ await flushTests()
+ }
+ })
+})
--- /dev/null
+import './config'
+import './email'
+import './follows'
+import './handle-down'
+import './jobs'
+import './reverse-proxy'
+import './stats'
+import './tracker'
--- /dev/null
+import './user-subscriptions'
+import './users'
+import './users-multiple-servers'
--- /dev/null
+import './multiple-servers'
+import './services'
+import './single-server'
+import './video-abuse'
+import './video-blacklist'
+import './video-blacklist-management'
+import './video-captions'
+import './video-channels'
+import './video-comme'
+import './video-description'
+import './video-impo'
+import './video-nsfw'
+import './video-privacy'
+import './video-schedule-update'
+import './video-transcoder'
--- /dev/null
+import './feeds'
// Order of the tests we want to execute
import './client'
import './activitypub'
-import './api/'
+import './feeds/'
import './cli/'
+import './api/'
projects / github / Chocobozzz / PeerTube.git / commitdiff
