Merge branch 'develop' into pr/1285

9 files changed, 170 insertions, 18 deletions

diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 7905d9ffa..29fdb263e 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -12,7 +12,7 @@ function checkMissedConfig () {
     'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
-    'storage.redundancy', 'storage.tmp',
+    'storage.redundancy', 'storage.tmp', 'storage.playlists',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'cache.previews.size', 'admin.email', 'contact_form.enabled',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 4a88aef87..3656a23f9 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -16,7 +16,7 @@ let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 315
+const LAST_MIGRATION_VERSION = 340
 
 // ---------------------------------------------------------------------------
 
@@ -192,6 +192,7 @@ const CONFIG = {
     AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
     LOG_DIR: buildPath(config.get<string>('storage.logs')),
     VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
+    PLAYLISTS_DIR: buildPath(config.get<string>('storage.playlists')),
     REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
     THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
     PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
@@ -259,6 +260,9 @@ const CONFIG = {
       get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') },
       get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') },
       get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
+    },
+    HLS: {
+      get ENABLED () { return config.get<boolean>('transcoding.hls.enabled') }
     }
   },
   IMPORT: {
@@ -316,8 +320,8 @@ let CONSTRAINTS_FIELDS = {
     BLOCKED_REASON: { min: 3, max: 250 } // Length
   },
   VIDEO_ABUSES: {
-    REASON: { min: 2, max: 300 }, // Length
-    MODERATION_COMMENT: { min: 2, max: 300 } // Length
+    REASON: { min: 2, max: 3000 }, // Length
+    MODERATION_COMMENT: { min: 2, max: 3000 } // Length
   },
   VIDEO_BLACKLIST: {
     REASON: { min: 2, max: 300 } // Length
@@ -590,6 +594,9 @@ const STATIC_PATHS = {
   TORRENTS: '/static/torrents/',
   WEBSEED: '/static/webseed/',
   REDUNDANCY: '/static/redundancy/',
+  PLAYLISTS: {
+    HLS: '/static/playlists/hls'
+  },
   AVATARS: '/static/avatars/',
   VIDEO_CAPTIONS: '/static/video-captions/'
 }
@@ -632,6 +639,9 @@ const CACHE = {
   }
 }
 
+const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls')
+const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
+
 const MEMOIZE_TTL = {
   OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
 }
@@ -701,6 +711,8 @@ if (isTestInstance() === true) {
   CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
   MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
   ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
+
+  RATES_LIMIT.LOGIN.MAX = 20
 }
 
 updateWebserverUrls()
@@ -709,6 +721,7 @@ updateWebserverUrls()
 
 export {
   API_VERSION,
+  HLS_REDUNDANCY_DIRECTORY,
   AVATARS_SIZE,
   ACCEPT_HEADERS,
   BCRYPT_SALT_SIZE,
@@ -733,6 +746,7 @@ export {
   PRIVATE_RSA_KEY_SIZE,
   ROUTE_CACHE_LIFETIME,
   SORTABLE_COLUMNS,
+  HLS_PLAYLIST_DIRECTORY,
   FEEDS,
   JOB_TTL,
   NSFW_POLICY_TYPES,
@@ -795,7 +809,9 @@ function buildVideoMimetypeExt () {
       'video/quicktime': '.mov',
       'video/x-msvideo': '.avi',
       'video/x-flv': '.flv',
-      'video/x-matroska': '.mkv'
+      'video/x-matroska': '.mkv',
+      'application/octet-stream': '.mkv',
+      'video/avi': '.avi'
     })
   }
 
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 84ad2079b..fe296142d 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist'
 import { ServerBlocklistModel } from '../models/server/server-blocklist'
 import { UserNotificationModel } from '../models/account/user-notification'
 import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
+import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) {
     AccountBlocklistModel,
     ServerBlocklistModel,
     UserNotificationModel,
-    UserNotificationSettingModel
+    UserNotificationSettingModel,
+    VideoStreamingPlaylistModel
   ])
 
   // Check extensions exist in the database
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index b9a9da183..2b22e16fe 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
 import { ApplicationModel } from '../models/application/application'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
 import { applicationExist, clientsExist, usersExist } from './checker-after-init'
-import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants'
+import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
 import { sequelizeTypescript } from './database'
 import { remove, ensureDir } from 'fs-extra'
 
@@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () {
     tasks.push(ensureDir(dir))
   }
 
+  // Playlist directories
+  tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY))
+
   return Promise.all(tasks)
 }
 
diff --git a/server/initializers/migrations/0320-blacklist-unfederate.ts b/server/initializers/migrations/0320-blacklist-unfederate.ts
new file mode 100644
index 000000000..6fb7bbb90
--- /dev/null
+++ b/server/initializers/migrations/0320-blacklist-unfederate.ts
@@ -0,0 +1,27 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+
+  {
+    const data = {
+      type: Sequelize.BOOLEAN,
+      allowNull: false,
+      defaultValue: false
+    }
+
+    await utils.queryInterface.addColumn('videoBlacklist', 'unfederated', data)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}
diff --git a/server/initializers/migrations/0325-video-abuse-fields.ts b/server/initializers/migrations/0325-video-abuse-fields.ts
new file mode 100644
index 000000000..fca6d666f
--- /dev/null
+++ b/server/initializers/migrations/0325-video-abuse-fields.ts
@@ -0,0 +1,37 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+
+  {
+    const data = {
+      type: Sequelize.STRING(3000),
+      allowNull: false,
+      defaultValue: null
+    }
+
+    await utils.queryInterface.changeColumn('videoAbuse', 'reason', data)
+  }
+
+  {
+    const data = {
+      type: Sequelize.STRING(3000),
+      allowNull: true,
+      defaultValue: null
+    }
+
+    await utils.queryInterface.changeColumn('videoAbuse', 'moderationComment', data)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}
diff --git a/server/initializers/migrations/0330-video-streaming-playlist.ts b/server/initializers/migrations/0330-video-streaming-playlist.ts
new file mode 100644
index 000000000..c85a762ab
--- /dev/null
+++ b/server/initializers/migrations/0330-video-streaming-playlist.ts
@@ -0,0 +1,51 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+
+  {
+    const query = `
+  CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist"
+(
+  "id"                       SERIAL,
+  "type"                     INTEGER                  NOT NULL,
+  "playlistUrl"              VARCHAR(2000)            NOT NULL,
+  "p2pMediaLoaderInfohashes" VARCHAR(255)[]           NOT NULL,
+  "segmentsSha256Url"        VARCHAR(255)             NOT NULL,
+  "videoId"                  INTEGER                  NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+  "createdAt"                TIMESTAMP WITH TIME ZONE NOT NULL,
+  "updatedAt"                TIMESTAMP WITH TIME ZONE NOT NULL,
+  PRIMARY KEY ("id")
+);`
+    await utils.sequelize.query(query)
+  }
+
+  {
+    const data = {
+      type: Sequelize.INTEGER,
+      allowNull: true,
+      defaultValue: null
+    }
+
+    await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data)
+  }
+
+  {
+    const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' +
+      'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE'
+
+    await utils.sequelize.query(query)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}
diff --git a/server/initializers/migrations/0335-video-downloading-enabled.ts b/server/initializers/migrations/0335-video-downloading-enabled.ts
new file mode 100644
index 000000000..e79466447
--- /dev/null
+++ b/server/initializers/migrations/0335-video-downloading-enabled.ts
@@ -0,0 +1,27 @@
+import * as Sequelize from 'sequelize'
+import { Migration } from '../../models/migrations'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+  const data = {
+    type: Sequelize.BOOLEAN,
+    allowNull: false,
+    defaultValue: true
+  } as Migration.Boolean
+  await utils.queryInterface.addColumn('video', 'downloadEnabled', data)
+
+  data.defaultValue = null
+  return utils.queryInterface.changeColumn('video', 'downloadEnabled', data)
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}
diff --git a/server/initializers/migrations/0295-add-originally-published-at.ts b/server/initializers/migrations/0340-add-originally-published-at.ts
index e02cca480..ab8d66854 100644
--- a/server/initializers/migrations/0295-add-originally-published-at.ts
+++ b/server/initializers/migrations/0340-add-originally-published-at.ts
@@ -19,17 +19,6 @@ async function up (utils: {
     const query = 'UPDATE video SET "originallyPublishedAt" = video."publishedAt"'
     await utils.sequelize.query(query)
   }
-
-  // Sequelize does not alter the column with NOW as default value
-  {
-    const data = {
-      type: Sequelize.DATE,
-      allowNull: false,
-      defaultValue: Sequelize.NOW
-    }
-    await utils.queryInterface.changeColumn('video', 'originallyPublishedAt', data)
-  }
-
 }
 
 function down (options) {