From a66c2e3252d6cca8958959966f42494ded564023 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 7 May 2021 08:59:59 +0200 Subject: Fix remote actor creation date --- server/initializers/constants.ts | 2 +- .../migrations/0645-actor-remote-creation-date.ts | 26 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 server/initializers/migrations/0645-actor-remote-creation-date.ts (limited to 'server/initializers') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index d390fd95e..f807a1e58 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 640 +const LAST_MIGRATION_VERSION = 645 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0645-actor-remote-creation-date.ts b/server/initializers/migrations/0645-actor-remote-creation-date.ts new file mode 100644 index 000000000..38b3b881c --- /dev/null +++ b/server/initializers/migrations/0645-actor-remote-creation-date.ts @@ -0,0 +1,26 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + { + const data = { + type: Sequelize.DATE, + defaultValue: null, + allowNull: true + } + await utils.queryInterface.addColumn('actor', 'remoteCreatedAt', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} -- cgit v1.2.3 From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/initializers/constants.ts | 6 +++++- server/initializers/installer.ts | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'server/initializers') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index f807a1e58..6f388420e 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -208,7 +208,8 @@ const SCHEDULER_INTERVALS_MS = { autoFollowIndexInstances: 60000 * 60 * 24, // 1 day removeOldViews: 60000 * 60 * 24, // 1 day removeOldHistory: 60000 * 60 * 24, // 1 day - updateInboxStats: 1000 * 60// 1 minute + updateInboxStats: 1000 * 60, // 1 minute + removeDanglingResumableUploads: 60000 * 60 * 16 // 16 hours } // --------------------------------------------------------------------------- @@ -285,6 +286,7 @@ const CONSTRAINTS_FIELDS = { LIKES: { min: 0 }, DISLIKES: { min: 0 }, FILE_SIZE: { min: -1 }, + PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB URL: { min: 3, max: 2000 } // Length }, VIDEO_PLAYLISTS: { @@ -645,6 +647,7 @@ const LRU_CACHE = { } } +const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads') const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') @@ -819,6 +822,7 @@ export { PEERTUBE_VERSION, LAZY_STATIC_PATHS, SEARCH_INDEX, + RESUMABLE_UPLOAD_DIRECTORY, HLS_REDUNDANCY_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, ACTOR_IMAGES_SIZE, diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index cb58454cb..8dcff64e2 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { applicationExist, clientsExist, usersExist } from './checker-after-init' -import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' +import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants' import { sequelizeTypescript } from './database' import { ensureDir, remove } from 'fs-extra' import { CONFIG } from './config' @@ -79,6 +79,9 @@ function createDirectoriesIfNotExist () { // Playlist directories tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY)) + // Resumable upload directory + tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY)) + return Promise.all(tasks) } -- cgit v1.2.3 From 7d9ba5c08999c6482f0bc5e0c09c6f55b7724090 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 May 2021 11:15:29 +0200 Subject: Cleanup models directory organization --- server/initializers/checker-after-init.ts | 2 +- server/initializers/database.ts | 14 +++++++------- server/initializers/installer.ts | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'server/initializers') diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index a93c8b7fd..911734fa0 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -7,7 +7,7 @@ import { RecentlyAddedStrategy } from '../../shared/models/redundancy' import { isProdInstance, isTestInstance, parseSemVersion } from '../helpers/core-utils' import { isArray } from '../helpers/custom-validators/misc' import { logger } from '../helpers/logger' -import { UserModel } from '../models/account/user' +import { UserModel } from '../models/user/user' import { ApplicationModel, getServerActor } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { CONFIG, isEmailEnabled } from './config' diff --git a/server/initializers/database.ts b/server/initializers/database.ts index edf12bc41..75a13ec8b 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -2,6 +2,9 @@ import { QueryTypes, Transaction } from 'sequelize' import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' import { TrackerModel } from '@server/models/server/tracker' import { VideoTrackerModel } from '@server/models/server/video-tracker' +import { UserModel } from '@server/models/user/user' +import { UserNotificationModel } from '@server/models/user/user-notification' +import { UserVideoHistoryModel } from '@server/models/user/user-video-history' import { isTestInstance } from '../helpers/core-utils' import { logger } from '../helpers/logger' import { AbuseModel } from '../models/abuse/abuse' @@ -11,13 +14,9 @@ import { VideoCommentAbuseModel } from '../models/abuse/video-comment-abuse' import { AccountModel } from '../models/account/account' import { AccountBlocklistModel } from '../models/account/account-blocklist' import { AccountVideoRateModel } from '../models/account/account-video-rate' -import { ActorImageModel } from '../models/account/actor-image' -import { UserModel } from '../models/account/user' -import { UserNotificationModel } from '../models/account/user-notification' -import { UserNotificationSettingModel } from '../models/account/user-notification-setting' -import { UserVideoHistoryModel } from '../models/account/user-video-history' -import { ActorModel } from '../models/activitypub/actor' -import { ActorFollowModel } from '../models/activitypub/actor-follow' +import { ActorModel } from '../models/actor/actor' +import { ActorFollowModel } from '../models/actor/actor-follow' +import { ActorImageModel } from '../models/actor/actor-image' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { OAuthTokenModel } from '../models/oauth/oauth-token' @@ -25,6 +24,7 @@ import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' import { PluginModel } from '../models/server/plugin' import { ServerModel } from '../models/server/server' import { ServerBlocklistModel } from '../models/server/server-blocklist' +import { UserNotificationSettingModel } from '../models/user/user-notification-setting' import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' import { TagModel } from '../models/video/tag' import { ThumbnailModel } from '../models/video/thumbnail' diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index 8dcff64e2..676f88653 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -2,7 +2,7 @@ import * as passwordGenerator from 'password-generator' import { UserRole } from '../../shared' import { logger } from '../helpers/logger' import { createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user' -import { UserModel } from '../models/account/user' +import { UserModel } from '../models/user/user' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { applicationExist, clientsExist, usersExist } from './checker-after-init' -- cgit v1.2.3 From aea0b0e7cde7495e60fe07b4444067f53d35ce3f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 May 2021 12:04:44 +0200 Subject: Inject server config in HTML --- server/initializers/constants.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server/initializers') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6f388420e..4cf7dcf0a 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -702,7 +702,8 @@ const CUSTOM_HTML_TAG_COMMENTS = { TITLE: '', DESCRIPTION: '', CUSTOM_CSS: '', - META_TAGS: '' + META_TAGS: '', + SERVER_CONFIG: '' } // --------------------------------------------------------------------------- -- cgit v1.2.3 From cb4bab61c19f2be7858f9cc6e8e234f04b1d504f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 May 2021 14:17:53 +0200 Subject: Fix logger warning level --- server/initializers/checker-after-init.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/initializers') diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index a93c8b7fd..5fd1af82f 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -132,7 +132,7 @@ function checkConfig () { // Broadcast message if (CONFIG.BROADCAST_MESSAGE.ENABLED) { const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL - const available = [ 'info', 'warning', 'error' ] + const available = [ 'info', 'warn', 'error' ] if (available.includes(currentLevel) === false) { return 'Broadcast message level should be ' + available.join(' or ') + ' instead of ' + currentLevel -- cgit v1.2.3 From 31a911190b27d93f6324da3d826f96c7fbc5deb6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 25 May 2021 10:08:29 +0200 Subject: Fix broadcast message log level --- server/initializers/checker-after-init.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/initializers') diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index d75c0af04..911734fa0 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -132,7 +132,7 @@ function checkConfig () { // Broadcast message if (CONFIG.BROADCAST_MESSAGE.ENABLED) { const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL - const available = [ 'info', 'warn', 'error' ] + const available = [ 'info', 'warning', 'error' ] if (available.includes(currentLevel) === false) { return 'Broadcast message level should be ' + available.join(' or ') + ' instead of ' + currentLevel -- cgit v1.2.3 From 2539932e16129992a2c0889b4ff527c265a8e2c7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 27 May 2021 15:59:55 +0200 Subject: Instance homepage support (#4007) * Prepare homepage parsers * Add ability to update instance hompage * Add ability to set homepage as landing page * Add homepage preview in admin * Dynamically update left menu for homepage * Inject home content in homepage * Add videos list and channel miniature custom markup * Remove unused elements in markup service --- server/initializers/constants.ts | 2 +- server/initializers/database.ts | 4 ++- .../migrations/0650-actor-custom-pages.ts | 33 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 server/initializers/migrations/0650-actor-custom-pages.ts (limited to 'server/initializers') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 4cf7dcf0a..919f9ea6e 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 645 +const LAST_MIGRATION_VERSION = 650 // --------------------------------------------------------------------------- diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 75a13ec8b..38e7a76d0 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -44,6 +44,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla import { VideoTagModel } from '../models/video/video-tag' import { VideoViewModel } from '../models/video/video-view' import { CONFIG } from './config' +import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -141,7 +142,8 @@ async function initDatabaseModels (silent: boolean) { ThumbnailModel, TrackerModel, VideoTrackerModel, - PluginModel + PluginModel, + ActorCustomPageModel ]) // Check extensions exist in the database diff --git a/server/initializers/migrations/0650-actor-custom-pages.ts b/server/initializers/migrations/0650-actor-custom-pages.ts new file mode 100644 index 000000000..1338327e8 --- /dev/null +++ b/server/initializers/migrations/0650-actor-custom-pages.ts @@ -0,0 +1,33 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + { + const query = ` + CREATE TABLE IF NOT EXISTS "actorCustomPage" ( + "id" serial, + "content" TEXT, + "type" varchar(255) NOT NULL, + "actorId" integer NOT NULL REFERENCES "actor" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp WITH time zone NOT NULL, + "updatedAt" timestamp WITH time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + + await utils.sequelize.query(query) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} -- cgit v1.2.3