import * as Bluebird from 'bluebird'
import { maxBy, minBy } from 'lodash'
import * as magnetUtil from 'magnet-uri'
-import { join } from 'path'
-import * as request from 'request'
-import * as sequelize from 'sequelize'
+import { basename } from 'path'
+import { Transaction } from 'sequelize/types'
+import { TrackerModel } from '@server/models/server/tracker'
import { VideoLiveModel } from '@server/models/video/video-live'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import {
ActivityUrlObject,
ActivityVideoUrlObject
} from '../../../shared/index'
-import { ActivityIconObject, VideoObject } from '../../../shared/models/activitypub/objects'
+import { ActivityTrackerUrlObject, VideoObject } from '../../../shared/models/activitypub/objects'
import { VideoPrivacy } from '../../../shared/models/videos'
import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
-import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
+import {
+ isAPVideoFileUrlMetadataObject,
+ isAPVideoTrackerUrlObject,
+ sanitizeAndCheckVideoTorrentObject
+} from '../../helpers/custom-validators/activitypub/videos'
import { isArray } from '../../helpers/custom-validators/misc'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
import {
ACTIVITY_PUB,
P2P_MEDIA_LOADER_PEER_VERSION,
PREVIEWS_SIZE,
REMOTE_SCHEME,
- STATIC_PATHS,
THUMBNAILS_SIZE
} from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
MChannelDefault,
MChannelId,
MStreamingPlaylist,
+ MStreamingPlaylistFilesVideo,
+ MStreamingPlaylistVideo,
MVideo,
MVideoAccountLight,
MVideoAccountLightBlacklistAllFiles,
MVideoFullLight,
MVideoId,
MVideoImmutable,
- MVideoThumbnail
+ MVideoThumbnail,
+ MVideoWithHost
} from '../../types/models'
import { MThumbnail } from '../../types/models/video/thumbnail'
import { FilteredModelAttributes } from '../../types/sequelize'
import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
import { setVideoTags } from '../video'
import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
+import { generateTorrentFileName } from '../video-paths'
import { getOrCreateActorAndServerAndModel } from './actor'
import { crawlCollectionPage } from './crawl'
import { sendCreateVideo, sendUpdateVideo } from './send'
import { addVideoComments } from './video-comments'
import { createRates } from './video-rates'
-async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
+async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
const video = videoArg as MVideoAP
if (
}
}
-async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoObject }> {
- const options = {
- uri: videoUrl,
- method: 'GET',
- json: true,
- activityPub: true
- }
-
+async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
logger.info('Fetching remote video %s.', videoUrl)
- const { response, body } = await doRequest<any>(options)
+ const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
logger.debug('Remote video JSON is not valid.', { body })
- return { response, videoObject: undefined }
+ return { statusCode, videoObject: undefined }
}
- return { response, videoObject: body }
+ return { statusCode, videoObject: body }
}
async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
const host = video.VideoChannel.Account.Actor.Server.host
const path = video.getDescriptionAPIPath()
- const options = {
- uri: REMOTE_SCHEME.HTTP + '://' + host + path,
- json: true
- }
+ const url = REMOTE_SCHEME.HTTP + '://' + host + path
- const { body } = await doRequest<any>(options)
- return body.description ? body.description : ''
+ const { body } = await doJSONRequest<any>(url)
+ return body.description || ''
}
function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
videoFieldsSave = video.toJSON()
- // Check actor has the right to update the video
- const videoChannel = video.VideoChannel
- if (videoChannel.Account.id !== account.id) {
- throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
+ // Check we can update the channel: we trust the remote server
+ const oldVideoChannel = video.VideoChannel
+
+ if (!oldVideoChannel.Actor.serverId || !channel.Actor.serverId) {
+ throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
+ }
+
+ if (oldVideoChannel.Actor.serverId !== channel.Actor.serverId) {
+ throw new Error('New channel ' + channel.Actor.url + ' is not on the same server than new channel ' + oldVideoChannel.Actor.url)
}
const to = overrideTo || videoObject.to
if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
- if (videoUpdated.getPreview()) {
- const previewUrl = getPreviewUrl(getPreviewFromIcons(videoObject), video)
+ const previewIcon = getPreviewFromIcons(videoObject)
+ if (videoUpdated.getPreview() && previewIcon) {
const previewModel = createPlaceholderThumbnail({
- fileUrl: previewUrl,
+ fileUrl: previewIcon.url,
video,
type: ThumbnailType.PREVIEW,
- size: PREVIEWS_SIZE
+ size: previewIcon
})
await videoUpdated.addAndSaveThumbnail(previewModel, t)
}
for (const playlistAttributes of streamingPlaylistAttributes) {
const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
- .then(([ streamingPlaylist ]) => streamingPlaylist)
+ .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo)
+ streamingPlaylistModel.Video = videoUpdated
const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
.map(a => new VideoFileModel(a))
const tags = videoObject.tag
.filter(isAPHashTagObject)
.map(tag => tag.name)
- await setVideoTags({ video: videoUpdated, tags, transaction: t, defaultValue: videoUpdated.Tags })
+ await setVideoTags({ video: videoUpdated, tags, transaction: t })
+ }
+
+ // Update trackers
+ {
+ const trackers = getTrackerUrls(videoObject, videoUpdated)
+ await setVideoTrackers({ video: videoUpdated, trackers, transaction: t })
}
{
: await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
try {
- const { response, videoObject } = await fetchRemoteVideo(video.url)
- if (response.statusCode === HttpStatusCode.NOT_FOUND_404) {
- logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
-
- // Video does not exist anymore
- await video.destroy()
- return undefined
- }
+ const { videoObject } = await fetchRemoteVideo(video.url)
if (videoObject === undefined) {
logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
return video
} catch (err) {
+ if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
+ logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
+
+ // Video does not exist anymore
+ await video.destroy()
+ return undefined
+ }
+
logger.warn('Cannot refresh video %s.', options.video.url, { err })
ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
}
-function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
+function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
return url && url.mediaType === 'application/x-mpegURL'
}
if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
- const previewUrl = getPreviewUrl(getPreviewFromIcons(videoObject), videoCreated)
- const previewModel = createPlaceholderThumbnail({
- fileUrl: previewUrl,
- video: videoCreated,
- type: ThumbnailType.PREVIEW,
- size: PREVIEWS_SIZE
- })
+ const previewIcon = getPreviewFromIcons(videoObject)
+ if (previewIcon) {
+ const previewModel = createPlaceholderThumbnail({
+ fileUrl: previewIcon.url,
+ video: videoCreated,
+ type: ThumbnailType.PREVIEW,
+ size: previewIcon
+ })
- if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
+ await videoCreated.addAndSaveThumbnail(previewModel, t)
+ }
// Process files
const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
videoCreated.VideoStreamingPlaylists = []
for (const playlistAttributes of streamingPlaylistsAttributes) {
- const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t })
+ const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo
+ playlist.Video = videoCreated
- const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject)
+ const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject)
const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
- playlistModel.VideoFiles = await Promise.all(videoFilePromises)
+ playlist.VideoFiles = await Promise.all(videoFilePromises)
- videoCreated.VideoStreamingPlaylists.push(playlistModel)
+ videoCreated.VideoStreamingPlaylists.push(playlist)
}
// Process tags
})
await Promise.all(videoCaptionsPromises)
+ // Process trackers
+ {
+ const trackers = getTrackerUrls(videoObject, videoCreated)
+ await setVideoTrackers({ video: videoCreated, trackers, transaction: t })
+ }
+
videoCreated.VideoFiles = videoFiles
if (videoCreated.isLive) {
}
function videoFileActivityUrlToDBAttributes (
- videoOrPlaylist: MVideo | MStreamingPlaylist,
+ videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
urls: (ActivityTagObject | ActivityUrlObject)[]
) {
const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
throw new Error('Cannot parse magnet URI ' + magnet.href)
}
+ const torrentUrl = Array.isArray(parsed.xs)
+ ? parsed.xs[0]
+ : parsed.xs
+
// Fetch associated metadata url, if any
- const metadata = urls.filter(isAPVideoFileMetadataObject)
+ const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
.find(u => {
return u.height === fileUrl.height &&
u.fps === fileUrl.fps &&
u.rel.includes(fileUrl.mediaType)
})
- const mediaType = fileUrl.mediaType
+ const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
+ const resolution = fileUrl.height
+ const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
+ const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
+
const attribute = {
- extname: getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, mediaType),
+ extname,
infoHash: parsed.infoHash,
- resolution: fileUrl.height,
+ resolution,
size: fileUrl.size,
fps: fileUrl.fps || -1,
metadataUrl: metadata?.href,
+ // Use the name of the remote file because we don't proxify video file requests
+ filename: basename(fileUrl.href),
+ fileUrl: fileUrl.href,
+
+ torrentUrl,
+ // Use our own torrent name since we proxify torrent requests
+ torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
+
// This is a video file owned by a video or by a streaming playlist
- videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id,
- videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
+ videoId,
+ videoStreamingPlaylistId
}
attributes.push(attribute)
return maxBy(validIcons, 'width')
}
-function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoAccountLight) {
- return previewIcon
- ? previewIcon.url
- : buildRemoteVideoBaseUrl(video, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
+function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
+ let wsFound = false
+
+ const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
+ .map((u: ActivityTrackerUrlObject) => {
+ if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true
+
+ return u.href
+ })
+
+ if (wsFound) return trackers
+
+ return [
+ buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
+ buildRemoteVideoBaseUrl(video, '/tracker/announce')
+ ]
+}
+
+async function setVideoTrackers (options: {
+ video: MVideo
+ trackers: string[]
+ transaction?: Transaction
+}) {
+ const { video, trackers, transaction } = options
+
+ const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
+
+ await video.$set('Trackers', trackerInstances, { transaction })
}
projects / github / Chocobozzz / PeerTube.git / blobdiff