Basic video redundancy implementation

7 files changed, 137 insertions, 57 deletions

diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index a9de11fb0..1304c7559 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -14,20 +14,24 @@ function activityPubContextify <T> (data: T) {
       'https://w3id.org/security/v1',
       {
         RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
+        pt: 'https://joinpeertube.org/ns',
+        schema: 'http://schema.org#',
         Hashtag: 'as:Hashtag',
-        uuid: 'http://schema.org/identifier',
-        category: 'http://schema.org/category',
-        licence: 'http://schema.org/license',
-        subtitleLanguage: 'http://schema.org/subtitleLanguage',
+        uuid: 'schema:identifier',
+        category: 'schema:category',
+        licence: 'schema:license',
+        subtitleLanguage: 'schema:subtitleLanguage',
         sensitive: 'as:sensitive',
-        language: 'http://schema.org/inLanguage',
-        views: 'http://schema.org/Number',
-        stats: 'http://schema.org/Number',
-        size: 'http://schema.org/Number',
-        fps: 'http://schema.org/Number',
-        commentsEnabled: 'http://schema.org/Boolean',
-        waitTranscoding: 'http://schema.org/Boolean',
-        support: 'http://schema.org/Text'
+        language: 'schema:inLanguage',
+        views: 'schema:Number',
+        stats: 'schema:Number',
+        size: 'schema:Number',
+        fps: 'schema:Number',
+        commentsEnabled: 'schema:Boolean',
+        waitTranscoding: 'schema:Boolean',
+        expires: 'schema:expires',
+        support: 'schema:Text',
+        CacheFile: 'pt:CacheFile'
       },
       {
         likes: {
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts
index 381a29e66..2562ead9b 100644
--- a/server/helpers/custom-validators/activitypub/activity.ts
+++ b/server/helpers/custom-validators/activitypub/activity.ts
@@ -1,7 +1,10 @@
 import * as validator from 'validator'
 import { Activity, ActivityType } from '../../../../shared/models/activitypub'
 import {
-  isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid,
+  isActorAcceptActivityValid,
+  isActorDeleteActivityValid,
+  isActorFollowActivityValid,
+  isActorRejectActivityValid,
   isActorUpdateActivityValid
 } from './actor'
 import { isAnnounceActivityValid } from './announce'
@@ -11,12 +14,13 @@ import { isUndoActivityValid } from './undo'
 import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments'
 import {
   isVideoFlagValid,
-  sanitizeAndCheckVideoTorrentCreateActivity,
   isVideoTorrentDeleteActivityValid,
+  sanitizeAndCheckVideoTorrentCreateActivity,
   sanitizeAndCheckVideoTorrentUpdateActivity
 } from './videos'
 import { isViewActivityValid } from './view'
 import { exists } from '../misc'
+import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file'
 
 function isRootActivityValid (activity: any) {
   return Array.isArray(activity['@context']) && (
@@ -67,11 +71,13 @@ function checkCreateActivity (activity: any) {
     isDislikeActivityValid(activity) ||
     sanitizeAndCheckVideoTorrentCreateActivity(activity) ||
     isVideoFlagValid(activity) ||
-    isVideoCommentCreateActivityValid(activity)
+    isVideoCommentCreateActivityValid(activity) ||
+    isCacheFileCreateActivityValid(activity)
 }
 
 function checkUpdateActivity (activity: any) {
-  return sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
+  return isCacheFileUpdateActivityValid(activity) ||
+    sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
     isActorUpdateActivityValid(activity)
 }
 
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
new file mode 100644
index 000000000..bd70934c8
--- /dev/null
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -0,0 +1,28 @@
+import { isActivityPubUrlValid, isBaseActivityValid } from './misc'
+import { isRemoteVideoUrlValid } from './videos'
+import { isDateValid, exists } from '../misc'
+import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
+
+function isCacheFileCreateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Create') &&
+    isCacheFileObjectValid(activity.object)
+}
+
+function isCacheFileUpdateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Update') &&
+    isCacheFileObjectValid(activity.object)
+}
+
+function isCacheFileObjectValid (object: CacheFileObject) {
+  return exists(object) &&
+    object.type === 'CacheFile' &&
+    isDateValid(object.expires) &&
+    isActivityPubUrlValid(object.object) &&
+    isRemoteVideoUrlValid(object.url)
+}
+
+export {
+  isCacheFileUpdateActivityValid,
+  isCacheFileCreateActivityValid,
+  isCacheFileObjectValid
+}
diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts
index 6c5c7abca..4e2c57f04 100644
--- a/server/helpers/custom-validators/activitypub/misc.ts
+++ b/server/helpers/custom-validators/activitypub/misc.ts
@@ -3,7 +3,7 @@ import { CONSTRAINTS_FIELDS } from '../../../initializers'
 import { isTestInstance } from '../../core-utils'
 import { exists } from '../misc'
 
-function isActivityPubUrlValid (url: string) {
+function isUrlValid (url: string) {
   const isURLOptions = {
     require_host: true,
     require_tld: true,
@@ -17,13 +17,18 @@ function isActivityPubUrlValid (url: string) {
     isURLOptions.require_tld = false
   }
 
-  return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
+  return exists(url) && validator.isURL('' + url, isURLOptions)
+}
+
+function isActivityPubUrlValid (url: string) {
+  return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
 }
 
 function isBaseActivityValid (activity: any, type: string) {
   return (activity['@context'] === undefined || Array.isArray(activity['@context'])) &&
     activity.type === type &&
     isActivityPubUrlValid(activity.id) &&
+    exists(activity.actor) &&
     (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) &&
     (
       activity.to === undefined ||
@@ -49,6 +54,7 @@ function setValidAttributedTo (obj: any) {
 }
 
 export {
+  isUrlValid,
   isActivityPubUrlValid,
   isBaseActivityValid,
   setValidAttributedTo
diff --git a/server/helpers/custom-validators/activitypub/undo.ts b/server/helpers/custom-validators/activitypub/undo.ts
index f50f224fa..578035893 100644
--- a/server/helpers/custom-validators/activitypub/undo.ts
+++ b/server/helpers/custom-validators/activitypub/undo.ts
@@ -2,6 +2,7 @@ import { isActorFollowActivityValid } from './actor'
 import { isBaseActivityValid } from './misc'
 import { isDislikeActivityValid, isLikeActivityValid } from './rate'
 import { isAnnounceActivityValid } from './announce'
+import { isCacheFileCreateActivityValid } from './cache-file'
 
 function isUndoActivityValid (activity: any) {
   return isBaseActivityValid(activity, 'Undo') &&
@@ -9,7 +10,8 @@ function isUndoActivityValid (activity: any) {
       isActorFollowActivityValid(activity.object) ||
       isLikeActivityValid(activity.object) ||
       isDislikeActivityValid(activity.object) ||
-      isAnnounceActivityValid(activity.object)
+      isAnnounceActivityValid(activity.object) ||
+      isCacheFileCreateActivityValid(activity.object)
     )
 }
 
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 0362f43ab..f76eba474 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -75,6 +75,30 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
     video.attributedTo.length !== 0
 }
 
+function isRemoteVideoUrlValid (url: any) {
+  // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11)
+  if (url.width && !url.height) url.height = url.width
+
+  return url.type === 'Link' &&
+    (
+      ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
+      isActivityPubUrlValid(url.href) &&
+      validator.isInt(url.height + '', { min: 0 }) &&
+      validator.isInt(url.size + '', { min: 0 }) &&
+      (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
+    ) ||
+    (
+      ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
+      isActivityPubUrlValid(url.href) &&
+      validator.isInt(url.height + '', { min: 0 })
+    ) ||
+    (
+      ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
+      validator.isLength(url.href, { min: 5 }) &&
+      validator.isInt(url.height + '', { min: 0 })
+    )
+}
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -83,7 +107,8 @@ export {
   isVideoTorrentDeleteActivityValid,
   isRemoteStringIdentifierValid,
   isVideoFlagValid,
-  sanitizeAndCheckVideoTorrentObject
+  sanitizeAndCheckVideoTorrentObject,
+  isRemoteVideoUrlValid
 }
 
 // ---------------------------------------------------------------------------
@@ -147,26 +172,4 @@ function setRemoteVideoTruncatedContent (video: any) {
   return true
 }
 
-function isRemoteVideoUrlValid (url: any) {
-  // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11)
-  if (url.width && !url.height) url.height = url.width
 
-  return url.type === 'Link' &&
-    (
-      ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
-      isActivityPubUrlValid(url.href) &&
-      validator.isInt(url.height + '', { min: 0 }) &&
-      validator.isInt(url.size + '', { min: 0 }) &&
-      (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
-    ) ||
-    (
-      ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
-      isActivityPubUrlValid(url.href) &&
-      validator.isInt(url.height + '', { min: 0 })
-    ) ||
-    (
-      ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
-      validator.isLength(url.href, { min: 5 }) &&
-      validator.isInt(url.height + '', { min: 0 })
-    )
-}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 1c0cc7058..2fdfd1876 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -5,44 +5,49 @@ import { createWriteStream, remove } from 'fs-extra'
 import { CONFIG } from '../initializers'
 import { join } from 'path'
 
-function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) {
+function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) {
   const id = target.magnetUri || target.torrentName
+  let timer
 
   const path = generateVideoTmpPath(id)
   logger.info('Importing torrent video %s', id)
 
   return new Promise<string>((res, rej) => {
     const webtorrent = new WebTorrent()
+    let file: WebTorrent.TorrentFile
 
     const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
 
     const options = { path: CONFIG.STORAGE.VIDEOS_DIR }
     const torrent = webtorrent.add(torrentId, options, torrent => {
-      if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId))
+      if (torrent.files.length !== 1) {
+        if (timer) clearTimeout(timer)
 
-      const file = torrent.files[ 0 ]
+        return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
+          .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId)))
+      }
+
+      file = torrent.files[ 0 ]
 
       const writeStream = createWriteStream(path)
       writeStream.on('finish', () => {
-        webtorrent.destroy(async err => {
-          if (err) return rej(err)
-
-          if (target.torrentName) {
-            remove(torrentId)
-              .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
-          }
+        if (timer) clearTimeout(timer)
 
-          remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name))
-            .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err }))
-
-          res(path)
-        })
+        return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
+          .then(() => res(path))
       })
 
       file.createReadStream().pipe(writeStream)
     })
 
     torrent.on('error', err => rej(err))
+
+    if (timeout) {
+      timer = setTimeout(async () => {
+        return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName)
+          .then(() => rej(new Error('Webtorrent download timeout.')))
+      }, timeout)
+    }
   })
 }
 
@@ -51,3 +56,29 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri
 export {
   downloadWebTorrentVideo
 }
+
+// ---------------------------------------------------------------------------
+
+function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) {
+  return new Promise(res => {
+    webtorrent.destroy(err => {
+      // Delete torrent file
+      if (torrentName) {
+        remove(torrentId)
+          .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
+      }
+
+      // Delete downloaded file
+      if (filename) {
+        remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename))
+          .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err }))
+      }
+
+      if (err) {
+        logger.warn('Cannot destroy webtorrent in timeout.', { err })
+      }
+
+      return res()
+    })
+  })
+}