From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- scripts/migrations/peertube-4.0.ts | 104 ------------------------------- scripts/migrations/peertube-4.2.ts | 124 ------------------------------------- scripts/migrations/peertube-5.0.ts | 71 --------------------- 3 files changed, 299 deletions(-) delete mode 100644 scripts/migrations/peertube-4.0.ts delete mode 100644 scripts/migrations/peertube-4.2.ts delete mode 100644 scripts/migrations/peertube-5.0.ts (limited to 'scripts/migrations') diff --git a/scripts/migrations/peertube-4.0.ts b/scripts/migrations/peertube-4.0.ts deleted file mode 100644 index b0891c2e6..000000000 --- a/scripts/migrations/peertube-4.0.ts +++ /dev/null @@ -1,104 +0,0 @@ -import Bluebird from 'bluebird' -import { move, readFile, writeFile } from 'fs-extra' -import { join } from 'path' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { JobQueue } from '@server/lib/job-queue' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoModel } from '@server/models/video/video' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { initDatabaseModels } from '../../server/initializers/database' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - console.log('Migrate old HLS paths to new format.') - - await initDatabaseModels(true) - - JobQueue.Instance.init() - - const ids = await VideoModel.listLocalIds() - - await Bluebird.map(ids, async id => { - try { - await processVideo(id) - } catch (err) { - console.error('Cannot process video %s.', { err }) - } - }, { concurrency: 5 }) - - console.log('Migration finished!') -} - -async function processVideo (videoId: number) { - const video = await VideoModel.loadWithFiles(videoId) - - const hls = video.getHLSPlaylist() - if (video.isLive || !hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) { - return - } - - console.log(`Renaming HLS playlist files of video ${video.name}.`) - - const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) - const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video) - - const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename) - let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8') - - for (const videoFile of hls.VideoFiles) { - const srcName = `${videoFile.resolution}.m3u8` - const dstName = getHlsResolutionPlaylistFilename(videoFile.filename) - - const src = join(hlsDirPath, srcName) - const dst = join(hlsDirPath, dstName) - - try { - await move(src, dst) - - masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName) - } catch (err) { - console.error('Cannot move video file %s to %s.', src, dst, err) - } - } - - await writeFile(masterPlaylistPath, masterPlaylistContent) - - if (playlist.segmentsSha256Filename === 'segments-sha256.json') { - try { - const newName = generateHlsSha256SegmentsFilename(video.isLive) - - const dst = join(hlsDirPath, newName) - await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst) - playlist.segmentsSha256Filename = newName - } catch (err) { - console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err) - } - } - - if (playlist.playlistFilename === 'master.m3u8') { - try { - const newName = generateHLSMasterPlaylistFilename(video.isLive) - - const dst = join(hlsDirPath, newName) - await move(join(hlsDirPath, playlist.playlistFilename), dst) - playlist.playlistFilename = newName - } catch (err) { - console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err) - } - } - - // Everything worked, we can save the playlist now - await playlist.save() - - const allVideo = await VideoModel.loadFull(video.id) - await federateVideoIfNeeded(allVideo, false) - - console.log(`Successfully moved HLS files of ${video.name}.`) -} diff --git a/scripts/migrations/peertube-4.2.ts b/scripts/migrations/peertube-4.2.ts deleted file mode 100644 index d8929692b..000000000 --- a/scripts/migrations/peertube-4.2.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { minBy } from 'lodash' -import { join } from 'path' -import { getImageSize, processImage } from '@server/helpers/image-utils' -import { CONFIG } from '@server/initializers/config' -import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' -import { updateActorImages } from '@server/lib/activitypub/actors' -import { sendUpdateActor } from '@server/lib/activitypub/send' -import { getBiggestActorImage } from '@server/lib/actor-image' -import { JobQueue } from '@server/lib/job-queue' -import { AccountModel } from '@server/models/account/account' -import { ActorModel } from '@server/models/actor/actor' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { ActorImageType } from '@shared/models' -import { initDatabaseModels } from '../../server/initializers/database' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - console.log('Generate avatar miniatures from existing avatars.') - - await initDatabaseModels(true) - JobQueue.Instance.init() - - const accounts: AccountModel[] = await AccountModel.findAll({ - include: [ - { - model: ActorModel, - required: true, - where: { - serverId: null - } - }, - { - model: VideoChannelModel, - include: [ - { - model: AccountModel - } - ] - } - ] - }) - - for (const account of accounts) { - try { - await fillAvatarSizeIfNeeded(account) - await generateSmallerAvatarIfNeeded(account) - } catch (err) { - console.error(`Cannot process account avatar ${account.name}`, err) - } - - for (const videoChannel of account.VideoChannels) { - try { - await fillAvatarSizeIfNeeded(videoChannel) - await generateSmallerAvatarIfNeeded(videoChannel) - } catch (err) { - console.error(`Cannot process channel avatar ${videoChannel.name}`, err) - } - } - } - - console.log('Generation finished!') -} - -async function fillAvatarSizeIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) { - const avatars = accountOrChannel.Actor.Avatars - - for (const avatar of avatars) { - if (avatar.width && avatar.height) continue - - console.log('Filling size of avatars of %s.', accountOrChannel.name) - - const { width, height } = await getImageSize(join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, avatar.filename)) - avatar.width = width - avatar.height = height - - await avatar.save() - } -} - -async function generateSmallerAvatarIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) { - const avatars = accountOrChannel.Actor.Avatars - if (avatars.length !== 1) { - return - } - - console.log(`Processing ${accountOrChannel.name}.`) - - await generateSmallerAvatar(accountOrChannel.Actor) - accountOrChannel.Actor = Object.assign(accountOrChannel.Actor, { Server: null }) - - return sendUpdateActor(accountOrChannel, undefined) -} - -async function generateSmallerAvatar (actor: MActorDefault) { - const bigAvatar = getBiggestActorImage(actor.Avatars) - - const imageSize = minBy(ACTOR_IMAGES_SIZE[ActorImageType.AVATAR], 'width') - const sourceFilename = bigAvatar.filename - - const newImageName = buildUUID() + getLowercaseExtension(sourceFilename) - const source = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, sourceFilename) - const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, newImageName) - - await processImage({ path: source, destination, newSize: imageSize, keepOriginal: true }) - - const actorImageInfo = { - name: newImageName, - fileUrl: null, - height: imageSize.height, - width: imageSize.width, - onDisk: true - } - - await updateActorImages(actor, ActorImageType.AVATAR, [ actorImageInfo ], undefined) -} diff --git a/scripts/migrations/peertube-5.0.ts b/scripts/migrations/peertube-5.0.ts deleted file mode 100644 index a0f51a64c..000000000 --- a/scripts/migrations/peertube-5.0.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { ensureDir } from 'fs-extra' -import { Op } from 'sequelize' -import { updateTorrentMetadata } from '@server/helpers/webtorrent' -import { DIRECTORIES } from '@server/initializers/constants' -import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy' -import { VideoModel } from '@server/models/video/video' -import { MVideoFullLight } from '@server/types/models' -import { VideoPrivacy } from '@shared/models' -import { initDatabaseModels } from '../../server/initializers/database' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - console.log('Moving private video files in dedicated folders.') - - await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) - await ensureDir(DIRECTORIES.VIDEOS.PRIVATE) - - await initDatabaseModels(true) - - const videos = await VideoModel.unscoped().findAll({ - attributes: [ 'uuid' ], - where: { - privacy: { - [Op.in]: [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ] - } - } - }) - - for (const { uuid } of videos) { - try { - console.log('Moving files of video %s.', uuid) - - const video = await VideoModel.loadFull(uuid) - - try { - await moveFilesIfPrivacyChanged(video, VideoPrivacy.PUBLIC) - } catch (err) { - console.error('Cannot move files of video %s.', uuid, err) - } - - try { - await updateTorrents(video) - } catch (err) { - console.error('Cannot regenerate torrents of video %s.', uuid, err) - } - } catch (err) { - console.error('Cannot process video %s.', uuid, err) - } - } -} - -async function updateTorrents (video: MVideoFullLight) { - for (const file of video.VideoFiles) { - await updateTorrentMetadata(video, file) - - await file.save() - } - - const playlist = video.getHLSPlaylist() - for (const file of (playlist?.VideoFiles || [])) { - await updateTorrentMetadata(playlist, file) - - await file.save() - } -} -- cgit v1.2.3