From c48e82b5e0478434de30626d14594a97f2402e7c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Sep 2018 16:27:07 +0200 Subject: Basic video redundancy implementation --- server/initializers/checker.ts | 17 ++++++++++ server/initializers/constants.ts | 36 ++++++++++++++++++++-- server/initializers/database.ts | 4 ++- .../migrations/0270-server-redundancy.ts | 24 +++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 server/initializers/migrations/0270-server-redundancy.ts (limited to 'server/initializers') diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 9dd104035..6a2badd35 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -7,6 +7,9 @@ import { parse } from 'url' import { CONFIG } from './constants' import { logger } from '../helpers/logger' import { getServerActor } from '../helpers/utils' +import { VideosRedundancy } from '../../shared/models/redundancy' +import { isArray } from '../helpers/custom-validators/misc' +import { uniq } from 'lodash' async function checkActivityPubUrls () { const actor = await getServerActor() @@ -35,6 +38,20 @@ function checkConfig () { return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy } + const redundancyVideos = config.get('redundancy.videos') + if (isArray(redundancyVideos)) { + for (const r of redundancyVideos) { + if ([ 'most-views' ].indexOf(r.strategy) === -1) { + return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy + } + } + + const filtered = uniq(redundancyVideos.map(r => r.strategy)) + if (filtered.length !== redundancyVideos.length) { + return 'Redundancy video entries should have uniq strategies' + } + } + return null } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5b7ea5d6c..6b4afbfd8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -1,6 +1,6 @@ import { IConfig } from 'config' import { dirname, join } from 'path' -import { JobType, VideoRateType, VideoState } from '../../shared/models' +import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' import { ActivityPubActorType } from '../../shared/models/activitypub' import { FollowState } from '../../shared/models/actors' import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' @@ -9,13 +9,14 @@ import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../h import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import { invert } from 'lodash' import { CronRepeatOptions, EveryRepeatOptions } from 'bull' +import * as bytes from 'bytes' // Use a variable to reload the configuration if we need let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 265 +const LAST_MIGRATION_VERSION = 270 // --------------------------------------------------------------------------- @@ -137,7 +138,8 @@ let SCHEDULER_INTERVALS_MS = { badActorFollow: 60000 * 60, // 1 hour removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute - youtubeDLUpdate: 60000 * 60 * 24 // 1 day + youtubeDLUpdate: 60000 * 60 * 24, // 1 day + videosRedundancy: 60000 * 2 // 2 hours } // --------------------------------------------------------------------------- @@ -208,6 +210,9 @@ const CONFIG = { INTERVAL_DAYS: config.get('trending.videos.interval_days') } }, + REDUNDANCY: { + VIDEOS: buildVideosRedundancy(config.get('redundancy.videos')) + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, @@ -321,6 +326,9 @@ const CONSTRAINTS_FIELDS = { } } }, + VIDEOS_REDUNDANCY: { + URL: { min: 3, max: 2000 } // Length + }, VIDEOS: { NAME: { min: 3, max: 120 }, // Length LANGUAGE: { min: 1, max: 10 }, // Length @@ -584,6 +592,13 @@ const CACHE = { } } +const REDUNDANCY = { + VIDEOS: { + EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days + RANDOMIZED_FACTOR: 5 + } +} + const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) // --------------------------------------------------------------------------- @@ -629,8 +644,11 @@ if (isTestInstance() === true) { SCHEDULER_INTERVALS_MS.badActorFollow = 10000 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 SCHEDULER_INTERVALS_MS.updateVideos = 5000 + SCHEDULER_INTERVALS_MS.videosRedundancy = 5000 REPEAT_JOBS['videos-views'] = { every: 5000 } + REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 + VIDEO_VIEW_LIFETIME = 1000 // 1 second JOB_ATTEMPTS['email'] = 1 @@ -653,6 +671,7 @@ export { CONFIG, CONSTRAINTS_FIELDS, EMBED_SIZE, + REDUNDANCY, JOB_CONCURRENCY, JOB_ATTEMPTS, LAST_MIGRATION_VERSION, @@ -722,6 +741,17 @@ function updateWebserverConfig () { CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) } +function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { + if (!objs) return [] + + return objs.map(obj => { + return { + strategy: obj.strategy, + size: bytes.parse(obj.size) + } + }) +} + function buildLanguages () { const iso639 = require('iso-639-3') diff --git a/server/initializers/database.ts b/server/initializers/database.ts index b68e1a882..4d57bf8aa 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -27,6 +27,7 @@ import { VideoCaptionModel } from '../models/video/video-caption' import { VideoImportModel } from '../models/video/video-import' import { VideoViewModel } from '../models/video/video-views' import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership' +import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -87,7 +88,8 @@ async function initDatabaseModels (silent: boolean) { VideoCommentModel, ScheduleVideoUpdateModel, VideoImportModel, - VideoViewModel + VideoViewModel, + VideoRedundancyModel ]) // Check extensions exist in the database diff --git a/server/initializers/migrations/0270-server-redundancy.ts b/server/initializers/migrations/0270-server-redundancy.ts new file mode 100644 index 000000000..903ba8a85 --- /dev/null +++ b/server/initializers/migrations/0270-server-redundancy.ts @@ -0,0 +1,24 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + { + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false + } + + await utils.queryInterface.addColumn('server', 'redundancyAllowed', data) + } + +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { up, down } -- cgit v1.2.3