From 2a491182e483b97afb1b65c908b23cb48d591807 Mon Sep 17 00:00:00 2001 From: Florent Date: Wed, 10 Aug 2022 09:53:39 +0200 Subject: Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz --- server/controllers/api/accounts.ts | 28 +++ server/controllers/api/config.ts | 4 + server/controllers/api/index.ts | 2 + server/controllers/api/server/debug.ts | 4 +- server/controllers/api/video-channel-sync.ts | 76 +++++++ server/controllers/api/video-channel.ts | 28 +++ server/controllers/api/videos/import.ts | 318 +++++---------------------- 7 files changed, 192 insertions(+), 268 deletions(-) create mode 100644 server/controllers/api/video-channel-sync.ts (limited to 'server/controllers') diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts index 66cdaab82..7a530cde5 100644 --- a/server/controllers/api/accounts.ts +++ b/server/controllers/api/accounts.ts @@ -25,8 +25,10 @@ import { accountsFollowersSortValidator, accountsSortValidator, ensureAuthUserOwnsAccountValidator, + ensureCanManageUser, videoChannelsSortValidator, videoChannelStatsValidator, + videoChannelSyncsSortValidator, videosSortValidator } from '../../middlewares/validators' import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists' @@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { VideoModel } from '../../models/video/video' import { VideoChannelModel } from '../../models/video/video-channel' import { VideoPlaylistModel } from '../../models/video/video-playlist' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' const accountsRouter = express.Router() @@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels', asyncMiddleware(listAccountChannels) ) +accountsRouter.get('/:accountName/video-channel-syncs', + authenticate, + asyncMiddleware(accountNameWithHostGetValidator), + ensureCanManageUser, + paginationValidator, + videoChannelSyncsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listAccountChannelsSync) +) + accountsRouter.get('/:accountName/video-playlists', optionalAuthenticate, asyncMiddleware(accountNameWithHostGetValidator), @@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response) return res.json(getFormattedObjects(resultList.data, resultList.total)) } +async function listAccountChannelsSync (req: express.Request, res: express.Response) { + const options = { + accountId: res.locals.account.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search + } + + const resultList = await VideoChannelSyncModel.listByAccountForAPI(options) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + async function listAccountPlaylists (req: express.Request, res: express.Response) { const serverActor = await getServerActor() diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index ff2fa9d86..f0fb43071 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -273,6 +273,10 @@ function customConfig (): CustomConfig { torrent: { enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED } + }, + videoChannelSynchronization: { + enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED, + maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER } }, trending: { diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index d1d4ef765..8c8ebd061 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts @@ -20,6 +20,7 @@ import { usersRouter } from './users' import { videoChannelRouter } from './video-channel' import { videoPlaylistRouter } from './video-playlist' import { videosRouter } from './videos' +import { videoChannelSyncRouter } from './video-channel-sync' const apiRouter = express.Router() @@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter) apiRouter.use('/users', usersRouter) apiRouter.use('/accounts', accountsRouter) apiRouter.use('/video-channels', videoChannelRouter) +apiRouter.use('/video-channel-syncs', videoChannelSyncRouter) apiRouter.use('/video-playlists', videoPlaylistRouter) apiRouter.use('/videos', videosRouter) apiRouter.use('/jobs', jobsRouter) diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index e09510dc3..4e5333782 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts @@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models' import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' import { UserRight } from '../../../../shared/models/users' import { authenticate, ensureUserHasRight } from '../../../middlewares' +import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' const debugRouter = express.Router() @@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) { const processors: { [id in SendDebugCommand['command']]: () => Promise } = { 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), - 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats() + 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), + 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() } await processors[body.command]() diff --git a/server/controllers/api/video-channel-sync.ts b/server/controllers/api/video-channel-sync.ts new file mode 100644 index 000000000..c2770b8e4 --- /dev/null +++ b/server/controllers/api/video-channel-sync.ts @@ -0,0 +1,76 @@ +import express from 'express' +import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger' +import { logger } from '@server/helpers/logger' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + ensureCanManageChannel as ensureCanManageSyncedChannel, + ensureSyncExists, + ensureSyncIsEnabled, + videoChannelSyncValidator +} from '@server/middlewares' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' +import { MChannelSyncFormattable } from '@server/types/models' +import { HttpStatusCode, VideoChannelSyncState } from '@shared/models' + +const videoChannelSyncRouter = express.Router() +const auditLogger = auditLoggerFactory('channel-syncs') + +videoChannelSyncRouter.post('/', + authenticate, + ensureSyncIsEnabled, + asyncMiddleware(videoChannelSyncValidator), + ensureCanManageSyncedChannel, + asyncRetryTransactionMiddleware(createVideoChannelSync) +) + +videoChannelSyncRouter.delete('/:id', + authenticate, + asyncMiddleware(ensureSyncExists), + ensureCanManageSyncedChannel, + asyncRetryTransactionMiddleware(removeVideoChannelSync) +) + +export { videoChannelSyncRouter } + +// --------------------------------------------------------------------------- + +async function createVideoChannelSync (req: express.Request, res: express.Response) { + const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({ + externalChannelUrl: req.body.externalChannelUrl, + videoChannelId: req.body.videoChannelId, + state: VideoChannelSyncState.WAITING_FIRST_RUN + }) + + await syncCreated.save() + syncCreated.VideoChannel = res.locals.videoChannel + + auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON())) + + logger.info( + 'Video synchronization for channel "%s" with external channel "%s" created.', + syncCreated.VideoChannel.name, + syncCreated.externalChannelUrl + ) + + return res.json({ + videoChannelSync: syncCreated.toFormattedJSON() + }) +} + +async function removeVideoChannelSync (req: express.Request, res: express.Response) { + const syncInstance = res.locals.videoChannelSync + + await syncInstance.destroy() + + auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON())) + + logger.info( + 'Video synchronization for channel "%s" with external channel "%s" deleted.', + syncInstance.VideoChannel.name, + syncInstance.externalChannelUrl + ) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 6b33e894d..89c7181bd 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -36,7 +36,9 @@ import { videoPlaylistsSortValidator } from '../../middlewares' import { + ensureChannelOwnerCanUpload, ensureIsLocalChannel, + videoChannelImportVideosValidator, videoChannelsFollowersSortValidator, videoChannelsListValidator, videoChannelsNameWithHostValidator, @@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers', asyncMiddleware(listVideoChannelFollowers) ) +videoChannelRouter.post('/:nameWithHost/import-videos', + authenticate, + asyncMiddleware(videoChannelsNameWithHostValidator), + videoChannelImportVideosValidator, + ensureIsLocalChannel, + ensureCanManageChannel, + asyncMiddleware(ensureChannelOwnerCanUpload), + asyncMiddleware(importVideosInChannel) +) + // --------------------------------------------------------------------------- export { @@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res return res.json(getFormattedObjects(resultList.data, resultList.total)) } + +async function importVideosInChannel (req: express.Request, res: express.Response) { + const { externalChannelUrl } = req.body + + await JobQueue.Instance.createJob({ + type: 'video-channel-import', + payload: { + externalChannelUrl, + videoChannelId: res.locals.videoChannel.id + } + }) + + logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 5a2e1006a..9d7b0260b 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -1,49 +1,20 @@ import express from 'express' -import { move, readFile, remove } from 'fs-extra' +import { move, readFile } from 'fs-extra' import { decode } from 'magnet-uri' import parseTorrent, { Instance } from 'parse-torrent' import { join } from 'path' -import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' -import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' -import { isResolvingToUnicastOnly } from '@server/helpers/dns' -import { Hooks } from '@server/lib/plugins/hooks' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { setVideoTags } from '@server/lib/video' -import { FilteredModelAttributes } from '@server/types' -import { - MChannelAccountDefault, - MThumbnail, - MUser, - MVideoAccountDefault, - MVideoCaption, - MVideoTag, - MVideoThumbnail, - MVideoWithBlacklistLight -} from '@server/types/models' -import { MVideoImportFormattable } from '@server/types/models/video/video-import' -import { - HttpStatusCode, - ServerErrorCode, - ThumbnailType, - VideoImportCreate, - VideoImportState, - VideoPrivacy, - VideoState -} from '@shared/models' +import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import' +import { MThumbnail, MVideoThumbnail } from '@server/types/models' +import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models' import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' -import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' import { isArray } from '../../../helpers/custom-validators/misc' import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' import { logger } from '../../../helpers/logger' import { getSecureTorrentName } from '../../../helpers/utils' -import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl' import { CONFIG } from '../../../initializers/config' import { MIMETYPES } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url' import { JobQueue } from '../../../lib/job-queue/job-queue' -import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' -import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' +import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail' import { asyncMiddleware, asyncRetryTransactionMiddleware, @@ -52,9 +23,6 @@ import { videoImportCancelValidator, videoImportDeleteValidator } from '../../../middlewares' -import { VideoModel } from '../../../models/video/video' -import { VideoCaptionModel } from '../../../models/video/video-caption' -import { VideoImportModel } from '../../../models/video/video-import' const auditLogger = auditLoggerFactory('video-imports') const videoImportsRouter = express.Router() @@ -68,7 +36,7 @@ videoImportsRouter.post('/imports', authenticate, reqVideoFileImport, asyncMiddleware(videoImportAddValidator), - asyncRetryTransactionMiddleware(addVideoImport) + asyncRetryTransactionMiddleware(handleVideoImport) ) videoImportsRouter.post('/imports/:id/cancel', @@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) { return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } -function addVideoImport (req: express.Request, res: express.Response) { - if (req.body.targetUrl) return addYoutubeDLImport(req, res) +function handleVideoImport (req: express.Request, res: express.Response) { + if (req.body.targetUrl) return handleYoutubeDlImport(req, res) const file = req.files?.['torrentfile']?.[0] - if (req.body.magnetUri || file) return addTorrentImport(req, res, file) + if (req.body.magnetUri || file) return handleTorrentImport(req, res, file) } -async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { +async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { const body: VideoImportCreate = req.body const user = res.locals.oauth.token.User @@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to videoName = result.name } - const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName }) + const video = await buildVideoFromImport({ + channelId: res.locals.videoChannel.id, + importData: { name: videoName }, + importDataOverride: body, + importType: 'torrent' + }) const thumbnailModel = await processThumbnail(req, video) const previewModel = await processPreview(req, video) - const videoImport = await insertIntoDB({ + const videoImport = await insertFromImportIntoDB({ video, thumbnailModel, previewModel, @@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to } }) - // Create job to import the video - const payload = { + const payload: VideoImportPayload = { type: torrentfile - ? 'torrent-file' as 'torrent-file' - : 'magnet-uri' as 'magnet-uri', + ? 'torrent-file' + : 'magnet-uri', videoImportId: videoImport.id, - magnetUri + preventException: false } await JobQueue.Instance.createJob({ type: 'video-import', payload }) @@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to return res.json(videoImport.toFormattedJSON()).end() } -async function addYoutubeDLImport (req: express.Request, res: express.Response) { +function statusFromYtDlImportError (err: YoutubeDlImportError): number { + switch (err.code) { + case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL: + return HttpStatusCode.FORBIDDEN_403 + + case YoutubeDlImportError.CODE.FETCH_ERROR: + return HttpStatusCode.BAD_REQUEST_400 + + default: + return HttpStatusCode.INTERNAL_SERVER_ERROR_500 + } +} + +async function handleYoutubeDlImport (req: express.Request, res: express.Response) { const body: VideoImportCreate = req.body const targetUrl = body.targetUrl const user = res.locals.oauth.token.User - const youtubeDL = new YoutubeDLWrapper( - targetUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - // Get video infos - let youtubeDLInfo: YoutubeDLInfo try { - youtubeDLInfo = await youtubeDL.getInfoForDownload() + const { job, videoImport } = await buildYoutubeDLImport({ + targetUrl, + channel: res.locals.videoChannel, + importDataOverride: body, + thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path, + previewFilePath: req.files?.['previewfile']?.[0].path, + user + }) + await JobQueue.Instance.createJob(job) + + auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) + + return res.json(videoImport.toFormattedJSON()).end() } catch (err) { - logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) + logger.error('An error occurred while importing the video %s. ', targetUrl, { err }) return res.fail({ - message: 'Cannot fetch remote information of this URL.', + message: err.message, + status: statusFromYtDlImportError(err), data: { targetUrl } }) } - - if (!await hasUnicastURLsOnly(youtubeDLInfo)) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot use non unicast IP as targetUrl.' - }) - } - - const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) - - // Process video thumbnail from request.files - let thumbnailModel = await processThumbnail(req, video) - - // Process video thumbnail from url if processing from request.files failed - if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) { - try { - thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video) - } catch (err) { - logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err }) - } - } - - // Process video preview from request.files - let previewModel = await processPreview(req, video) - - // Process video preview from url if processing from request.files failed - if (!previewModel && youtubeDLInfo.thumbnailUrl) { - try { - previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video) - } catch (err) { - logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err }) - } - } - - const videoImport = await insertIntoDB({ - video, - thumbnailModel, - previewModel, - videoChannel: res.locals.videoChannel, - tags: body.tags || youtubeDLInfo.tags, - user, - videoImportAttributes: { - targetUrl, - state: VideoImportState.PENDING, - userId: user.id - } - }) - - // Get video subtitles - await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) - - let fileExt = `.${youtubeDLInfo.ext}` - if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' - - // Create job to import the video - const payload = { - type: 'youtube-dl' as 'youtube-dl', - videoImportId: videoImport.id, - fileExt - } - await JobQueue.Instance.createJob({ type: 'video-import', payload }) - - auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) - - return res.json(videoImport.toFormattedJSON()).end() -} - -async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise { - let videoData = { - name: body.name || importData.name || 'Unknown name', - remote: false, - category: body.category || importData.category, - licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, - language: body.language || importData.language, - commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, - downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, - waitTranscoding: body.waitTranscoding || false, - state: VideoState.TO_IMPORT, - nsfw: body.nsfw || importData.nsfw || false, - description: body.description || importData.description, - support: body.support || null, - privacy: body.privacy || VideoPrivacy.PRIVATE, - duration: 0, // duration will be set by the import job - channelId, - originallyPublishedAt: body.originallyPublishedAt - ? new Date(body.originallyPublishedAt) - : importData.originallyPublishedAt - } - - videoData = await Hooks.wrapObject( - videoData, - body.targetUrl - ? 'filter:api.video.import-url.video-attribute.result' - : 'filter:api.video.import-torrent.video-attribute.result' - ) - - const video = new VideoModel(videoData) - video.url = getLocalVideoActivityPubUrl(video) - - return video } async function processThumbnail (req: express.Request, video: MVideoThumbnail) { @@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr return undefined } -async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) { - try { - return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE }) - } catch (err) { - logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err }) - return undefined - } -} - -async function processPreviewFromUrl (url: string, video: MVideoThumbnail) { - try { - return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW }) - } catch (err) { - logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err }) - return undefined - } -} - -async function insertIntoDB (parameters: { - video: MVideoThumbnail - thumbnailModel: MThumbnail - previewModel: MThumbnail - videoChannel: MChannelAccountDefault - tags: string[] - videoImportAttributes: FilteredModelAttributes - user: MUser -}): Promise { - const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters - - const videoImport = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - // Save video object in database - const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) - videoCreated.VideoChannel = videoChannel - - if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) - if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) - - await autoBlacklistVideoIfNeeded({ - video: videoCreated, - user, - notify: false, - isRemote: false, - isNew: true, - transaction: t - }) - - await setVideoTags({ video: videoCreated, tags, transaction: t }) - - // Create video import object in database - const videoImport = await VideoImportModel.create( - Object.assign({ videoId: videoCreated.id }, videoImportAttributes), - sequelizeOptions - ) as MVideoImportFormattable - videoImport.Video = videoCreated - - return videoImport - }) - - return videoImport -} - async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { const torrentName = torrentfile.originalname @@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) { function extractNameFromArray (name: string | string[]) { return isArray(name) ? name[0] : name } - -async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { - try { - const subtitles = await youtubeDL.getSubtitles() - - logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl) - - for (const subtitle of subtitles) { - if (!await isVTTFileValid(subtitle.path)) { - await remove(subtitle.path) - continue - } - - const videoCaption = new VideoCaptionModel({ - videoId, - language: subtitle.language, - filename: VideoCaptionModel.generateCaptionName(subtitle.language) - }) as MVideoCaption - - // Move physical file - await moveAndProcessCaptionFile(subtitle, videoCaption) - - await sequelizeTypescript.transaction(async t => { - await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) - }) - } - } catch (err) { - logger.warn('Cannot get video subtitles.', { err }) - } -} - -async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { - const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) - const uniqHosts = new Set(hosts) - - for (const h of uniqHosts) { - if (await isResolvingToUnicastOnly(h) !== true) { - return false - } - } - - return true -} -- cgit v1.2.3