From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- server/lib/activitypub/activity.ts | 74 --- server/lib/activitypub/actors/get.ts | 143 ----- server/lib/activitypub/actors/image.ts | 112 ---- server/lib/activitypub/actors/index.ts | 6 - server/lib/activitypub/actors/keys.ts | 16 - server/lib/activitypub/actors/refresh.ts | 81 --- server/lib/activitypub/actors/shared/creator.ts | 149 ----- server/lib/activitypub/actors/shared/index.ts | 3 - .../actors/shared/object-to-model-attributes.ts | 84 --- .../lib/activitypub/actors/shared/url-to-object.ts | 56 -- server/lib/activitypub/actors/updater.ts | 91 --- server/lib/activitypub/actors/webfinger.ts | 67 --- server/lib/activitypub/audience.ts | 34 -- server/lib/activitypub/cache-file.ts | 82 --- server/lib/activitypub/collection.ts | 63 -- server/lib/activitypub/context.ts | 212 ------- server/lib/activitypub/crawl.ts | 58 -- server/lib/activitypub/follow.ts | 51 -- server/lib/activitypub/inbox-manager.ts | 47 -- server/lib/activitypub/local-video-viewer.ts | 44 -- server/lib/activitypub/outbox.ts | 24 - server/lib/activitypub/playlists/create-update.ts | 157 ----- server/lib/activitypub/playlists/get.ts | 35 -- server/lib/activitypub/playlists/index.ts | 3 - server/lib/activitypub/playlists/refresh.ts | 53 -- server/lib/activitypub/playlists/shared/index.ts | 2 - .../playlists/shared/object-to-model-attributes.ts | 40 -- .../activitypub/playlists/shared/url-to-object.ts | 47 -- server/lib/activitypub/process/index.ts | 1 - server/lib/activitypub/process/process-accept.ts | 32 - server/lib/activitypub/process/process-announce.ts | 75 --- server/lib/activitypub/process/process-create.ts | 170 ------ server/lib/activitypub/process/process-delete.ts | 153 ----- server/lib/activitypub/process/process-dislike.ts | 58 -- server/lib/activitypub/process/process-flag.ts | 103 ---- server/lib/activitypub/process/process-follow.ts | 156 ----- server/lib/activitypub/process/process-like.ts | 60 -- server/lib/activitypub/process/process-reject.ts | 33 - server/lib/activitypub/process/process-undo.ts | 183 ------ server/lib/activitypub/process/process-update.ts | 119 ---- server/lib/activitypub/process/process-view.ts | 42 -- server/lib/activitypub/process/process.ts | 92 --- server/lib/activitypub/send/http.ts | 73 --- server/lib/activitypub/send/index.ts | 10 - server/lib/activitypub/send/send-accept.ts | 47 -- server/lib/activitypub/send/send-announce.ts | 58 -- server/lib/activitypub/send/send-create.ts | 226 ------- server/lib/activitypub/send/send-delete.ts | 158 ----- server/lib/activitypub/send/send-dislike.ts | 40 -- server/lib/activitypub/send/send-flag.ts | 42 -- server/lib/activitypub/send/send-follow.ts | 37 -- server/lib/activitypub/send/send-like.ts | 40 -- server/lib/activitypub/send/send-reject.ts | 39 -- server/lib/activitypub/send/send-undo.ts | 172 ------ server/lib/activitypub/send/send-update.ts | 157 ----- server/lib/activitypub/send/send-view.ts | 62 -- .../lib/activitypub/send/shared/audience-utils.ts | 74 --- server/lib/activitypub/send/shared/index.ts | 2 - server/lib/activitypub/send/shared/send-utils.ts | 291 --------- server/lib/activitypub/share.ts | 120 ---- server/lib/activitypub/url.ts | 177 ------ server/lib/activitypub/video-comments.ts | 205 ------- server/lib/activitypub/video-rates.ts | 59 -- server/lib/activitypub/videos/federate.ts | 29 - server/lib/activitypub/videos/get.ts | 116 ---- server/lib/activitypub/videos/index.ts | 4 - server/lib/activitypub/videos/refresh.ts | 68 --- .../activitypub/videos/shared/abstract-builder.ts | 190 ------ server/lib/activitypub/videos/shared/creator.ts | 65 -- server/lib/activitypub/videos/shared/index.ts | 6 - .../videos/shared/object-to-model-attributes.ts | 285 --------- server/lib/activitypub/videos/shared/trackers.ts | 43 -- .../lib/activitypub/videos/shared/url-to-object.ts | 25 - .../videos/shared/video-sync-attributes.ts | 107 ---- server/lib/activitypub/videos/updater.ts | 180 ------ server/lib/actor-follow-health-cache.ts | 86 --- server/lib/actor-image.ts | 14 - server/lib/auth/external-auth.ts | 231 ------- server/lib/auth/oauth-model.ts | 294 --------- server/lib/auth/oauth.ts | 223 ------- server/lib/auth/tokens-cache.ts | 52 -- server/lib/blocklist.ts | 62 -- server/lib/client-html.ts | 623 ------------------- server/lib/emailer.ts | 284 --------- server/lib/emails/abuse-new-message/html.pug | 11 - server/lib/emails/abuse-state-change/html.pug | 9 - server/lib/emails/account-abuse-new/html.pug | 14 - server/lib/emails/common/base.pug | 258 -------- server/lib/emails/common/greetings.pug | 11 - server/lib/emails/common/html.pug | 4 - server/lib/emails/common/mixins.pug | 7 - server/lib/emails/contact-form/html.pug | 9 - server/lib/emails/follower-on-channel/html.pug | 9 - server/lib/emails/password-create/html.pug | 10 - server/lib/emails/password-reset/html.pug | 12 - server/lib/emails/peertube-version-new/html.pug | 9 - server/lib/emails/plugin-version-new/html.pug | 9 - server/lib/emails/user-registered/html.pug | 10 - .../user-registration-request-accepted/html.pug | 10 - .../user-registration-request-rejected/html.pug | 9 - .../lib/emails/user-registration-request/html.pug | 9 - server/lib/emails/verify-email/html.pug | 19 - server/lib/emails/video-abuse-new/html.pug | 18 - .../lib/emails/video-auto-blacklist-new/html.pug | 17 - server/lib/emails/video-comment-abuse-new/html.pug | 16 - server/lib/emails/video-comment-mention/html.pug | 11 - server/lib/emails/video-comment-new/html.pug | 11 - .../lib/files-cache/avatar-permanent-file-cache.ts | 27 - server/lib/files-cache/index.ts | 6 - .../shared/abstract-permanent-file-cache.ts | 132 ---- .../shared/abstract-simple-file-cache.ts | 30 - server/lib/files-cache/shared/index.ts | 2 - .../video-captions-simple-file-cache.ts | 61 -- .../video-miniature-permanent-file-cache.ts | 28 - .../video-previews-simple-file-cache.ts | 58 -- .../video-storyboards-simple-file-cache.ts | 53 -- .../video-torrents-simple-file-cache.ts | 70 --- server/lib/hls.ts | 285 --------- server/lib/internal-event-emitter.ts | 35 -- .../lib/job-queue/handlers/activitypub-cleaner.ts | 202 ------- .../lib/job-queue/handlers/activitypub-follow.ts | 82 --- .../handlers/activitypub-http-broadcast.ts | 49 -- .../job-queue/handlers/activitypub-http-fetcher.ts | 41 -- .../job-queue/handlers/activitypub-http-unicast.ts | 38 -- .../job-queue/handlers/activitypub-refresher.ts | 60 -- server/lib/job-queue/handlers/actor-keys.ts | 20 - .../handlers/after-video-channel-import.ts | 37 -- server/lib/job-queue/handlers/email.ts | 17 - server/lib/job-queue/handlers/federate-video.ts | 28 - .../lib/job-queue/handlers/generate-storyboard.ts | 163 ----- .../lib/job-queue/handlers/manage-video-torrent.ts | 110 ---- .../job-queue/handlers/move-to-object-storage.ts | 159 ----- server/lib/job-queue/handlers/notify.ts | 27 - .../job-queue/handlers/transcoding-job-builder.ts | 48 -- .../lib/job-queue/handlers/video-channel-import.ts | 43 -- server/lib/job-queue/handlers/video-file-import.ts | 83 --- server/lib/job-queue/handlers/video-import.ts | 344 ----------- server/lib/job-queue/handlers/video-live-ending.ts | 279 --------- server/lib/job-queue/handlers/video-redundancy.ts | 17 - .../lib/job-queue/handlers/video-studio-edition.ts | 180 ------ server/lib/job-queue/handlers/video-transcoding.ts | 150 ----- server/lib/job-queue/handlers/video-views-stats.ts | 57 -- server/lib/job-queue/index.ts | 1 - server/lib/job-queue/job-queue.ts | 537 ----------------- server/lib/live/index.ts | 4 - server/lib/live/live-manager.ts | 552 ----------------- server/lib/live/live-quota-store.ts | 48 -- server/lib/live/live-segment-sha-store.ts | 95 --- server/lib/live/live-utils.ts | 99 --- server/lib/live/shared/index.ts | 1 - server/lib/live/shared/muxing-session.ts | 518 ---------------- .../abstract-transcoding-wrapper.ts | 110 ---- .../ffmpeg-transcoding-wrapper.ts | 107 ---- .../lib/live/shared/transcoding-wrapper/index.ts | 3 - .../remote-transcoding-wrapper.ts | 21 - server/lib/local-actor.ts | 102 ---- server/lib/model-loaders/actor.ts | 17 - server/lib/model-loaders/index.ts | 2 - server/lib/model-loaders/video.ts | 66 -- server/lib/moderation.ts | 258 -------- server/lib/notifier/index.ts | 1 - server/lib/notifier/notifier.ts | 284 --------- .../shared/abuse/abstract-new-abuse-message.ts | 67 --- .../abuse/abuse-state-change-for-reporter.ts | 74 --- server/lib/notifier/shared/abuse/index.ts | 4 - .../shared/abuse/new-abuse-for-moderators.ts | 119 ---- .../abuse/new-abuse-message-for-moderators.ts | 32 - .../shared/abuse/new-abuse-message-for-reporter.ts | 36 -- server/lib/notifier/shared/blacklist/index.ts | 3 - .../blacklist/new-auto-blacklist-for-moderators.ts | 60 -- .../shared/blacklist/new-blacklist-for-owner.ts | 58 -- .../shared/blacklist/unblacklist-for-owner.ts | 55 -- .../lib/notifier/shared/comment/comment-mention.ts | 111 ---- server/lib/notifier/shared/comment/index.ts | 2 - .../shared/comment/new-comment-for-video-owner.ts | 76 --- .../shared/common/abstract-notification.ts | 23 - server/lib/notifier/shared/common/index.ts | 1 - .../shared/follow/auto-follow-for-instance.ts | 51 -- .../notifier/shared/follow/follow-for-instance.ts | 68 --- .../lib/notifier/shared/follow/follow-for-user.ts | 82 --- server/lib/notifier/shared/follow/index.ts | 3 - server/lib/notifier/shared/index.ts | 7 - .../instance/direct-registration-for-moderators.ts | 49 -- server/lib/notifier/shared/instance/index.ts | 4 - .../instance/new-peertube-version-for-admins.ts | 54 -- .../instance/new-plugin-version-for-admins.ts | 58 -- .../registration-request-for-moderators.ts | 48 -- .../abstract-owned-video-publication.ts | 57 -- .../video-publication/import-finished-for-owner.ts | 97 --- .../lib/notifier/shared/video-publication/index.ts | 6 - .../video-publication/new-video-for-subscribers.ts | 61 -- .../owned-publication-after-auto-unblacklist.ts | 11 - .../owned-publication-after-schedule-update.ts | 10 - .../owned-publication-after-transcoding.ts | 9 - .../studio-edition-finished-for-owner.ts | 57 -- server/lib/object-storage/index.ts | 5 - server/lib/object-storage/keys.ts | 20 - server/lib/object-storage/pre-signed-urls.ts | 46 -- server/lib/object-storage/proxy.ts | 97 --- server/lib/object-storage/shared/client.ts | 71 --- server/lib/object-storage/shared/index.ts | 3 - server/lib/object-storage/shared/logger.ts | 7 - .../shared/object-storage-helpers.ts | 328 ---------- server/lib/object-storage/urls.ts | 63 -- server/lib/object-storage/videos.ts | 197 ------ .../bittorrent-tracker-observers-builder.ts | 51 -- server/lib/opentelemetry/metric-helpers/index.ts | 7 - .../metric-helpers/job-queue-observers-builder.ts | 24 - .../metric-helpers/lives-observers-builder.ts | 21 - .../metric-helpers/nodejs-observers-builder.ts | 202 ------- .../metric-helpers/playback-metrics.ts | 85 --- .../metric-helpers/stats-observers-builder.ts | 186 ------ .../metric-helpers/viewers-observers-builder.ts | 24 - server/lib/opentelemetry/metrics.ts | 123 ---- server/lib/opentelemetry/tracing.ts | 94 --- server/lib/paths.ts | 92 --- server/lib/peertube-socket.ts | 129 ---- server/lib/plugins/hooks.ts | 35 -- server/lib/plugins/plugin-helpers-builder.ts | 262 -------- server/lib/plugins/plugin-index.ts | 85 --- server/lib/plugins/plugin-manager.ts | 665 --------------------- server/lib/plugins/register-helpers.ts | 340 ----------- server/lib/plugins/theme-utils.ts | 24 - .../lib/plugins/video-constant-manager-factory.ts | 139 ----- server/lib/plugins/yarn.ts | 73 --- server/lib/redis.ts | 465 -------------- server/lib/redundancy.ts | 59 -- server/lib/runners/index.ts | 3 - .../runners/job-handlers/abstract-job-handler.ts | 269 --------- .../abstract-vod-transcoding-job-handler.ts | 66 -- server/lib/runners/job-handlers/index.ts | 7 - .../live-rtmp-hls-transcoding-job-handler.ts | 173 ------ .../runners/job-handlers/runner-job-handlers.ts | 20 - server/lib/runners/job-handlers/shared/index.ts | 1 - .../lib/runners/job-handlers/shared/vod-helpers.ts | 44 -- .../video-studio-transcoding-job-handler.ts | 157 ----- .../vod-audio-merge-transcoding-job-handler.ts | 97 --- .../vod-hls-transcoding-job-handler.ts | 114 ---- .../vod-web-video-transcoding-job-handler.ts | 84 --- server/lib/runners/runner-urls.ts | 13 - server/lib/runners/runner.ts | 49 -- server/lib/schedulers/abstract-scheduler.ts | 35 -- server/lib/schedulers/actor-follow-scheduler.ts | 54 -- .../lib/schedulers/auto-follow-index-instances.ts | 75 --- server/lib/schedulers/geo-ip-update-scheduler.ts | 22 - .../schedulers/peertube-version-check-scheduler.ts | 55 -- server/lib/schedulers/plugins-check-scheduler.ts | 74 --- .../remove-dangling-resumable-uploads-scheduler.ts | 40 -- .../lib/schedulers/remove-old-history-scheduler.ts | 31 - .../lib/schedulers/remove-old-views-scheduler.ts | 31 - .../schedulers/runner-job-watch-dog-scheduler.ts | 42 -- server/lib/schedulers/update-videos-scheduler.ts | 89 --- .../video-channel-sync-latest-scheduler.ts | 50 -- .../lib/schedulers/video-views-buffer-scheduler.ts | 52 -- .../lib/schedulers/videos-redundancy-scheduler.ts | 375 ------------ .../lib/schedulers/youtube-dl-update-scheduler.ts | 22 - server/lib/search.ts | 49 -- server/lib/server-config-manager.ts | 384 ------------ server/lib/signup.ts | 75 --- server/lib/stat-manager.ts | 182 ------ server/lib/sync-channel.ts | 111 ---- server/lib/thumbnail.ts | 327 ---------- server/lib/timeserie.ts | 61 -- server/lib/transcoding/create-transcoding-job.ts | 37 -- .../transcoding/default-transcoding-profiles.ts | 143 ----- server/lib/transcoding/ended-transcoding.ts | 18 - server/lib/transcoding/hls-transcoding.ts | 180 ------ server/lib/transcoding/shared/ffmpeg-builder.ts | 18 - server/lib/transcoding/shared/index.ts | 2 - .../shared/job-builders/abstract-job-builder.ts | 21 - .../lib/transcoding/shared/job-builders/index.ts | 2 - .../job-builders/transcoding-job-queue-builder.ts | 322 ---------- .../job-builders/transcoding-runner-job-builder.ts | 196 ------ server/lib/transcoding/transcoding-priority.ts | 24 - .../lib/transcoding/transcoding-quick-transcode.ts | 12 - server/lib/transcoding/transcoding-resolutions.ts | 73 --- server/lib/transcoding/web-transcoding.ts | 263 -------- server/lib/uploadx.ts | 37 -- server/lib/user.ts | 301 ---------- server/lib/video-blacklist.ts | 145 ----- server/lib/video-channel.ts | 50 -- server/lib/video-comment.ts | 116 ---- server/lib/video-file.ts | 145 ----- server/lib/video-path-manager.ts | 174 ------ server/lib/video-playlist.ts | 30 - server/lib/video-pre-import.ts | 323 ---------- server/lib/video-privacy.ts | 133 ----- server/lib/video-state.ts | 154 ----- server/lib/video-studio.ts | 130 ---- server/lib/video-tokens-manager.ts | 78 --- server/lib/video-urls.ts | 31 - server/lib/video.ts | 189 ------ server/lib/views/shared/index.ts | 3 - server/lib/views/shared/video-viewer-counters.ts | 198 ------ server/lib/views/shared/video-viewer-stats.ts | 196 ------ server/lib/views/shared/video-views.ts | 70 --- server/lib/views/video-views-manager.ts | 100 ---- server/lib/worker/parent-process.ts | 77 --- server/lib/worker/workers/http-broadcast.ts | 32 - server/lib/worker/workers/image-downloader.ts | 35 -- server/lib/worker/workers/image-processor.ts | 7 - 301 files changed, 27409 deletions(-) delete mode 100644 server/lib/activitypub/activity.ts delete mode 100644 server/lib/activitypub/actors/get.ts delete mode 100644 server/lib/activitypub/actors/image.ts delete mode 100644 server/lib/activitypub/actors/index.ts delete mode 100644 server/lib/activitypub/actors/keys.ts delete mode 100644 server/lib/activitypub/actors/refresh.ts delete mode 100644 server/lib/activitypub/actors/shared/creator.ts delete mode 100644 server/lib/activitypub/actors/shared/index.ts delete mode 100644 server/lib/activitypub/actors/shared/object-to-model-attributes.ts delete mode 100644 server/lib/activitypub/actors/shared/url-to-object.ts delete mode 100644 server/lib/activitypub/actors/updater.ts delete mode 100644 server/lib/activitypub/actors/webfinger.ts delete mode 100644 server/lib/activitypub/audience.ts delete mode 100644 server/lib/activitypub/cache-file.ts delete mode 100644 server/lib/activitypub/collection.ts delete mode 100644 server/lib/activitypub/context.ts delete mode 100644 server/lib/activitypub/crawl.ts delete mode 100644 server/lib/activitypub/follow.ts delete mode 100644 server/lib/activitypub/inbox-manager.ts delete mode 100644 server/lib/activitypub/local-video-viewer.ts delete mode 100644 server/lib/activitypub/outbox.ts delete mode 100644 server/lib/activitypub/playlists/create-update.ts delete mode 100644 server/lib/activitypub/playlists/get.ts delete mode 100644 server/lib/activitypub/playlists/index.ts delete mode 100644 server/lib/activitypub/playlists/refresh.ts delete mode 100644 server/lib/activitypub/playlists/shared/index.ts delete mode 100644 server/lib/activitypub/playlists/shared/object-to-model-attributes.ts delete mode 100644 server/lib/activitypub/playlists/shared/url-to-object.ts delete mode 100644 server/lib/activitypub/process/index.ts delete mode 100644 server/lib/activitypub/process/process-accept.ts delete mode 100644 server/lib/activitypub/process/process-announce.ts delete mode 100644 server/lib/activitypub/process/process-create.ts delete mode 100644 server/lib/activitypub/process/process-delete.ts delete mode 100644 server/lib/activitypub/process/process-dislike.ts delete mode 100644 server/lib/activitypub/process/process-flag.ts delete mode 100644 server/lib/activitypub/process/process-follow.ts delete mode 100644 server/lib/activitypub/process/process-like.ts delete mode 100644 server/lib/activitypub/process/process-reject.ts delete mode 100644 server/lib/activitypub/process/process-undo.ts delete mode 100644 server/lib/activitypub/process/process-update.ts delete mode 100644 server/lib/activitypub/process/process-view.ts delete mode 100644 server/lib/activitypub/process/process.ts delete mode 100644 server/lib/activitypub/send/http.ts delete mode 100644 server/lib/activitypub/send/index.ts delete mode 100644 server/lib/activitypub/send/send-accept.ts delete mode 100644 server/lib/activitypub/send/send-announce.ts delete mode 100644 server/lib/activitypub/send/send-create.ts delete mode 100644 server/lib/activitypub/send/send-delete.ts delete mode 100644 server/lib/activitypub/send/send-dislike.ts delete mode 100644 server/lib/activitypub/send/send-flag.ts delete mode 100644 server/lib/activitypub/send/send-follow.ts delete mode 100644 server/lib/activitypub/send/send-like.ts delete mode 100644 server/lib/activitypub/send/send-reject.ts delete mode 100644 server/lib/activitypub/send/send-undo.ts delete mode 100644 server/lib/activitypub/send/send-update.ts delete mode 100644 server/lib/activitypub/send/send-view.ts delete mode 100644 server/lib/activitypub/send/shared/audience-utils.ts delete mode 100644 server/lib/activitypub/send/shared/index.ts delete mode 100644 server/lib/activitypub/send/shared/send-utils.ts delete mode 100644 server/lib/activitypub/share.ts delete mode 100644 server/lib/activitypub/url.ts delete mode 100644 server/lib/activitypub/video-comments.ts delete mode 100644 server/lib/activitypub/video-rates.ts delete mode 100644 server/lib/activitypub/videos/federate.ts delete mode 100644 server/lib/activitypub/videos/get.ts delete mode 100644 server/lib/activitypub/videos/index.ts delete mode 100644 server/lib/activitypub/videos/refresh.ts delete mode 100644 server/lib/activitypub/videos/shared/abstract-builder.ts delete mode 100644 server/lib/activitypub/videos/shared/creator.ts delete mode 100644 server/lib/activitypub/videos/shared/index.ts delete mode 100644 server/lib/activitypub/videos/shared/object-to-model-attributes.ts delete mode 100644 server/lib/activitypub/videos/shared/trackers.ts delete mode 100644 server/lib/activitypub/videos/shared/url-to-object.ts delete mode 100644 server/lib/activitypub/videos/shared/video-sync-attributes.ts delete mode 100644 server/lib/activitypub/videos/updater.ts delete mode 100644 server/lib/actor-follow-health-cache.ts delete mode 100644 server/lib/actor-image.ts delete mode 100644 server/lib/auth/external-auth.ts delete mode 100644 server/lib/auth/oauth-model.ts delete mode 100644 server/lib/auth/oauth.ts delete mode 100644 server/lib/auth/tokens-cache.ts delete mode 100644 server/lib/blocklist.ts delete mode 100644 server/lib/client-html.ts delete mode 100644 server/lib/emailer.ts delete mode 100644 server/lib/emails/abuse-new-message/html.pug delete mode 100644 server/lib/emails/abuse-state-change/html.pug delete mode 100644 server/lib/emails/account-abuse-new/html.pug delete mode 100644 server/lib/emails/common/base.pug delete mode 100644 server/lib/emails/common/greetings.pug delete mode 100644 server/lib/emails/common/html.pug delete mode 100644 server/lib/emails/common/mixins.pug delete mode 100644 server/lib/emails/contact-form/html.pug delete mode 100644 server/lib/emails/follower-on-channel/html.pug delete mode 100644 server/lib/emails/password-create/html.pug delete mode 100644 server/lib/emails/password-reset/html.pug delete mode 100644 server/lib/emails/peertube-version-new/html.pug delete mode 100644 server/lib/emails/plugin-version-new/html.pug delete mode 100644 server/lib/emails/user-registered/html.pug delete mode 100644 server/lib/emails/user-registration-request-accepted/html.pug delete mode 100644 server/lib/emails/user-registration-request-rejected/html.pug delete mode 100644 server/lib/emails/user-registration-request/html.pug delete mode 100644 server/lib/emails/verify-email/html.pug delete mode 100644 server/lib/emails/video-abuse-new/html.pug delete mode 100644 server/lib/emails/video-auto-blacklist-new/html.pug delete mode 100644 server/lib/emails/video-comment-abuse-new/html.pug delete mode 100644 server/lib/emails/video-comment-mention/html.pug delete mode 100644 server/lib/emails/video-comment-new/html.pug delete mode 100644 server/lib/files-cache/avatar-permanent-file-cache.ts delete mode 100644 server/lib/files-cache/index.ts delete mode 100644 server/lib/files-cache/shared/abstract-permanent-file-cache.ts delete mode 100644 server/lib/files-cache/shared/abstract-simple-file-cache.ts delete mode 100644 server/lib/files-cache/shared/index.ts delete mode 100644 server/lib/files-cache/video-captions-simple-file-cache.ts delete mode 100644 server/lib/files-cache/video-miniature-permanent-file-cache.ts delete mode 100644 server/lib/files-cache/video-previews-simple-file-cache.ts delete mode 100644 server/lib/files-cache/video-storyboards-simple-file-cache.ts delete mode 100644 server/lib/files-cache/video-torrents-simple-file-cache.ts delete mode 100644 server/lib/hls.ts delete mode 100644 server/lib/internal-event-emitter.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-cleaner.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-follow.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-http-broadcast.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-http-fetcher.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-http-unicast.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-refresher.ts delete mode 100644 server/lib/job-queue/handlers/actor-keys.ts delete mode 100644 server/lib/job-queue/handlers/after-video-channel-import.ts delete mode 100644 server/lib/job-queue/handlers/email.ts delete mode 100644 server/lib/job-queue/handlers/federate-video.ts delete mode 100644 server/lib/job-queue/handlers/generate-storyboard.ts delete mode 100644 server/lib/job-queue/handlers/manage-video-torrent.ts delete mode 100644 server/lib/job-queue/handlers/move-to-object-storage.ts delete mode 100644 server/lib/job-queue/handlers/notify.ts delete mode 100644 server/lib/job-queue/handlers/transcoding-job-builder.ts delete mode 100644 server/lib/job-queue/handlers/video-channel-import.ts delete mode 100644 server/lib/job-queue/handlers/video-file-import.ts delete mode 100644 server/lib/job-queue/handlers/video-import.ts delete mode 100644 server/lib/job-queue/handlers/video-live-ending.ts delete mode 100644 server/lib/job-queue/handlers/video-redundancy.ts delete mode 100644 server/lib/job-queue/handlers/video-studio-edition.ts delete mode 100644 server/lib/job-queue/handlers/video-transcoding.ts delete mode 100644 server/lib/job-queue/handlers/video-views-stats.ts delete mode 100644 server/lib/job-queue/index.ts delete mode 100644 server/lib/job-queue/job-queue.ts delete mode 100644 server/lib/live/index.ts delete mode 100644 server/lib/live/live-manager.ts delete mode 100644 server/lib/live/live-quota-store.ts delete mode 100644 server/lib/live/live-segment-sha-store.ts delete mode 100644 server/lib/live/live-utils.ts delete mode 100644 server/lib/live/shared/index.ts delete mode 100644 server/lib/live/shared/muxing-session.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/index.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts delete mode 100644 server/lib/local-actor.ts delete mode 100644 server/lib/model-loaders/actor.ts delete mode 100644 server/lib/model-loaders/index.ts delete mode 100644 server/lib/model-loaders/video.ts delete mode 100644 server/lib/moderation.ts delete mode 100644 server/lib/notifier/index.ts delete mode 100644 server/lib/notifier/notifier.ts delete mode 100644 server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts delete mode 100644 server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts delete mode 100644 server/lib/notifier/shared/abuse/index.ts delete mode 100644 server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts delete mode 100644 server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts delete mode 100644 server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts delete mode 100644 server/lib/notifier/shared/blacklist/index.ts delete mode 100644 server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts delete mode 100644 server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts delete mode 100644 server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts delete mode 100644 server/lib/notifier/shared/comment/comment-mention.ts delete mode 100644 server/lib/notifier/shared/comment/index.ts delete mode 100644 server/lib/notifier/shared/comment/new-comment-for-video-owner.ts delete mode 100644 server/lib/notifier/shared/common/abstract-notification.ts delete mode 100644 server/lib/notifier/shared/common/index.ts delete mode 100644 server/lib/notifier/shared/follow/auto-follow-for-instance.ts delete mode 100644 server/lib/notifier/shared/follow/follow-for-instance.ts delete mode 100644 server/lib/notifier/shared/follow/follow-for-user.ts delete mode 100644 server/lib/notifier/shared/follow/index.ts delete mode 100644 server/lib/notifier/shared/index.ts delete mode 100644 server/lib/notifier/shared/instance/direct-registration-for-moderators.ts delete mode 100644 server/lib/notifier/shared/instance/index.ts delete mode 100644 server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts delete mode 100644 server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts delete mode 100644 server/lib/notifier/shared/instance/registration-request-for-moderators.ts delete mode 100644 server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts delete mode 100644 server/lib/notifier/shared/video-publication/import-finished-for-owner.ts delete mode 100644 server/lib/notifier/shared/video-publication/index.ts delete mode 100644 server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts delete mode 100644 server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts delete mode 100644 server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts delete mode 100644 server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts delete mode 100644 server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts delete mode 100644 server/lib/object-storage/index.ts delete mode 100644 server/lib/object-storage/keys.ts delete mode 100644 server/lib/object-storage/pre-signed-urls.ts delete mode 100644 server/lib/object-storage/proxy.ts delete mode 100644 server/lib/object-storage/shared/client.ts delete mode 100644 server/lib/object-storage/shared/index.ts delete mode 100644 server/lib/object-storage/shared/logger.ts delete mode 100644 server/lib/object-storage/shared/object-storage-helpers.ts delete mode 100644 server/lib/object-storage/urls.ts delete mode 100644 server/lib/object-storage/videos.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/index.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/playback-metrics.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metrics.ts delete mode 100644 server/lib/opentelemetry/tracing.ts delete mode 100644 server/lib/paths.ts delete mode 100644 server/lib/peertube-socket.ts delete mode 100644 server/lib/plugins/hooks.ts delete mode 100644 server/lib/plugins/plugin-helpers-builder.ts delete mode 100644 server/lib/plugins/plugin-index.ts delete mode 100644 server/lib/plugins/plugin-manager.ts delete mode 100644 server/lib/plugins/register-helpers.ts delete mode 100644 server/lib/plugins/theme-utils.ts delete mode 100644 server/lib/plugins/video-constant-manager-factory.ts delete mode 100644 server/lib/plugins/yarn.ts delete mode 100644 server/lib/redis.ts delete mode 100644 server/lib/redundancy.ts delete mode 100644 server/lib/runners/index.ts delete mode 100644 server/lib/runners/job-handlers/abstract-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/index.ts delete mode 100644 server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/runner-job-handlers.ts delete mode 100644 server/lib/runners/job-handlers/shared/index.ts delete mode 100644 server/lib/runners/job-handlers/shared/vod-helpers.ts delete mode 100644 server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts delete mode 100644 server/lib/runners/runner-urls.ts delete mode 100644 server/lib/runners/runner.ts delete mode 100644 server/lib/schedulers/abstract-scheduler.ts delete mode 100644 server/lib/schedulers/actor-follow-scheduler.ts delete mode 100644 server/lib/schedulers/auto-follow-index-instances.ts delete mode 100644 server/lib/schedulers/geo-ip-update-scheduler.ts delete mode 100644 server/lib/schedulers/peertube-version-check-scheduler.ts delete mode 100644 server/lib/schedulers/plugins-check-scheduler.ts delete mode 100644 server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts delete mode 100644 server/lib/schedulers/remove-old-history-scheduler.ts delete mode 100644 server/lib/schedulers/remove-old-views-scheduler.ts delete mode 100644 server/lib/schedulers/runner-job-watch-dog-scheduler.ts delete mode 100644 server/lib/schedulers/update-videos-scheduler.ts delete mode 100644 server/lib/schedulers/video-channel-sync-latest-scheduler.ts delete mode 100644 server/lib/schedulers/video-views-buffer-scheduler.ts delete mode 100644 server/lib/schedulers/videos-redundancy-scheduler.ts delete mode 100644 server/lib/schedulers/youtube-dl-update-scheduler.ts delete mode 100644 server/lib/search.ts delete mode 100644 server/lib/server-config-manager.ts delete mode 100644 server/lib/signup.ts delete mode 100644 server/lib/stat-manager.ts delete mode 100644 server/lib/sync-channel.ts delete mode 100644 server/lib/thumbnail.ts delete mode 100644 server/lib/timeserie.ts delete mode 100644 server/lib/transcoding/create-transcoding-job.ts delete mode 100644 server/lib/transcoding/default-transcoding-profiles.ts delete mode 100644 server/lib/transcoding/ended-transcoding.ts delete mode 100644 server/lib/transcoding/hls-transcoding.ts delete mode 100644 server/lib/transcoding/shared/ffmpeg-builder.ts delete mode 100644 server/lib/transcoding/shared/index.ts delete mode 100644 server/lib/transcoding/shared/job-builders/abstract-job-builder.ts delete mode 100644 server/lib/transcoding/shared/job-builders/index.ts delete mode 100644 server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts delete mode 100644 server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts delete mode 100644 server/lib/transcoding/transcoding-priority.ts delete mode 100644 server/lib/transcoding/transcoding-quick-transcode.ts delete mode 100644 server/lib/transcoding/transcoding-resolutions.ts delete mode 100644 server/lib/transcoding/web-transcoding.ts delete mode 100644 server/lib/uploadx.ts delete mode 100644 server/lib/user.ts delete mode 100644 server/lib/video-blacklist.ts delete mode 100644 server/lib/video-channel.ts delete mode 100644 server/lib/video-comment.ts delete mode 100644 server/lib/video-file.ts delete mode 100644 server/lib/video-path-manager.ts delete mode 100644 server/lib/video-playlist.ts delete mode 100644 server/lib/video-pre-import.ts delete mode 100644 server/lib/video-privacy.ts delete mode 100644 server/lib/video-state.ts delete mode 100644 server/lib/video-studio.ts delete mode 100644 server/lib/video-tokens-manager.ts delete mode 100644 server/lib/video-urls.ts delete mode 100644 server/lib/video.ts delete mode 100644 server/lib/views/shared/index.ts delete mode 100644 server/lib/views/shared/video-viewer-counters.ts delete mode 100644 server/lib/views/shared/video-viewer-stats.ts delete mode 100644 server/lib/views/shared/video-views.ts delete mode 100644 server/lib/views/video-views-manager.ts delete mode 100644 server/lib/worker/parent-process.ts delete mode 100644 server/lib/worker/workers/http-broadcast.ts delete mode 100644 server/lib/worker/workers/image-downloader.ts delete mode 100644 server/lib/worker/workers/image-processor.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts deleted file mode 100644 index 391bcd9c6..000000000 --- a/server/lib/activitypub/activity.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { doJSONRequest, PeerTubeRequestOptions } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { ActivityObject, ActivityPubActor, ActivityType, APObjectId } from '@shared/models' -import { buildSignedRequestOptions } from './send' - -export function getAPId (object: string | { id: string }) { - if (typeof object === 'string') return object - - return object.id -} - -export function getActivityStreamDuration (duration: number) { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return 'PT' + duration + 'S' -} - -export function getDurationFromActivityStream (duration: string) { - return parseInt(duration.replace(/[^\d]+/, '')) -} - -// --------------------------------------------------------------------------- - -export function buildAvailableActivities (): ActivityType[] { - return [ - 'Create', - 'Update', - 'Delete', - 'Follow', - 'Accept', - 'Announce', - 'Undo', - 'Like', - 'Reject', - 'View', - 'Dislike', - 'Flag' - ] -} - -// --------------------------------------------------------------------------- - -export async function fetchAP (url: string, moreOptions: PeerTubeRequestOptions = {}) { - const options = { - activityPub: true, - - httpSignature: CONFIG.FEDERATION.SIGN_FEDERATED_FETCHES - ? await buildSignedRequestOptions({ hasPayload: false }) - : undefined, - - ...moreOptions - } - - return doJSONRequest(url, options) -} - -export async function fetchAPObjectIfNeeded (object: APObjectId) { - if (typeof object === 'string') { - const { body } = await fetchAP>(object) - - return body - } - - return object as Exclude -} - -export async function findLatestAPRedirection (url: string, iteration = 1) { - if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) - - const { headers } = await fetchAP(url, { followRedirect: false }) - - if (headers.location) return findLatestAPRedirection(headers.location, iteration + 1) - - return url -} diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts deleted file mode 100644 index dd2bc9f03..000000000 --- a/server/lib/activitypub/actors/get.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { JobQueue } from '@server/lib/job-queue' -import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders' -import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' -import { arrayify } from '@shared/core-utils' -import { ActivityPubActor, APObjectId } from '@shared/models' -import { fetchAPObjectIfNeeded, getAPId } from '../activity' -import { checkUrlsSameHost } from '../url' -import { refreshActorIfNeeded } from './refresh' -import { APActorCreator, fetchRemoteActor } from './shared' - -function getOrCreateAPActor ( - activityActor: string | ActivityPubActor, - fetchType: 'all', - recurseIfNeeded?: boolean, - updateCollections?: boolean -): Promise - -function getOrCreateAPActor ( - activityActor: string | ActivityPubActor, - fetchType?: 'association-ids', - recurseIfNeeded?: boolean, - updateCollections?: boolean -): Promise - -async function getOrCreateAPActor ( - activityActor: string | ActivityPubActor, - fetchType: ActorLoadByUrlType = 'association-ids', - recurseIfNeeded = true, - updateCollections = false -): Promise { - const actorUrl = getAPId(activityActor) - let actor = await loadActorFromDB(actorUrl, fetchType) - - let created = false - let accountPlaylistsUrl: string - - // We don't have this actor in our database, fetch it on remote - if (!actor) { - const { actorObject } = await fetchRemoteActor(actorUrl) - if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) - - // actorUrl is just an alias/redirection, so process object id instead - if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections) - - // Create the attributed to actor - // In PeerTube a video channel is owned by an account - let ownerActor: MActorFullActor - if (recurseIfNeeded === true && actorObject.type === 'Group') { - ownerActor = await getOrCreateAPOwner(actorObject, actorUrl) - } - - const creator = new APActorCreator(actorObject, ownerActor) - actor = await retryTransactionWrapper(creator.create.bind(creator)) - created = true - accountPlaylistsUrl = actorObject.playlists - } - - if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor - if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor - - const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType }) - if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') - - await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) - await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl) - - return actorRefreshed -} - -async function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { - const accountAttributedTo = await findOwner(actorUrl, actorObject.attributedTo, 'Person') - if (!accountAttributedTo) { - throw new Error(`Cannot find account attributed to video channel ${actorUrl}`) - } - - try { - // Don't recurse another time - const recurseIfNeeded = false - return getOrCreateAPActor(accountAttributedTo, 'all', recurseIfNeeded) - } catch (err) { - logger.error('Cannot get or create account attributed to video channel ' + actorUrl) - throw new Error(err) - } -} - -async function findOwner (rootUrl: string, attributedTo: APObjectId[] | APObjectId, type: 'Person' | 'Group') { - for (const actorToCheck of arrayify(attributedTo)) { - const actorObject = await fetchAPObjectIfNeeded(getAPId(actorToCheck)) - - if (!actorObject) { - logger.warn('Unknown attributed to actor %s for owner %s', actorToCheck, rootUrl) - continue - } - - if (checkUrlsSameHost(actorObject.id, rootUrl) !== true) { - logger.warn(`Account attributed to ${actorObject.id} does not have the same host than owner actor url ${rootUrl}`) - continue - } - - if (actorObject.type === type) return actorObject - } - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - getOrCreateAPOwner, - getOrCreateAPActor, - findOwner -} - -// --------------------------------------------------------------------------- - -async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType) { - let actor = await loadActorByUrl(actorUrl, fetchType) - - // Orphan actor (not associated to an account of channel) so recreate it - if (actor && (!actor.Account && !actor.VideoChannel)) { - await actor.destroy() - actor = null - } - - return actor -} - -async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) { - if ((created === true || refreshed === true) && updateCollections === true) { - const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } - await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) - } -} - -async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) { - // We created a new account: fetch the playlists - if (created === true && actor.Account && accountPlaylistsUrl) { - const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' } - await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) - } -} diff --git a/server/lib/activitypub/actors/image.ts b/server/lib/activitypub/actors/image.ts deleted file mode 100644 index e1d29af5b..000000000 --- a/server/lib/activitypub/actors/image.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { logger } from '@server/helpers/logger' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { MActorImage, MActorImages } from '@server/types/models' -import { ActorImageType } from '@shared/models' - -type ImageInfo = { - name: string - fileUrl: string - height: number - width: number - onDisk?: boolean -} - -async function updateActorImages (actor: MActorImages, type: ActorImageType, imagesInfo: ImageInfo[], t: Transaction) { - const getAvatarsOrBanners = () => { - const result = type === ActorImageType.AVATAR - ? actor.Avatars - : actor.Banners - - return result || [] - } - - if (imagesInfo.length === 0) { - await deleteActorImages(actor, type, t) - } - - // Cleanup old images that did not have a width - for (const oldImageModel of getAvatarsOrBanners()) { - if (oldImageModel.width) continue - - await safeDeleteActorImage(actor, oldImageModel, type, t) - } - - for (const imageInfo of imagesInfo) { - const oldImageModel = getAvatarsOrBanners().find(i => imageInfo.width && i.width === imageInfo.width) - - if (oldImageModel) { - // Don't update the avatar if the file URL did not change - if (imageInfo.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) { - continue - } - - await safeDeleteActorImage(actor, oldImageModel, type, t) - } - - const imageModel = await ActorImageModel.create({ - filename: imageInfo.name, - onDisk: imageInfo.onDisk ?? false, - fileUrl: imageInfo.fileUrl, - height: imageInfo.height, - width: imageInfo.width, - type, - actorId: actor.id - }, { transaction: t }) - - addActorImage(actor, type, imageModel) - } - - return actor -} - -async function deleteActorImages (actor: MActorImages, type: ActorImageType, t: Transaction) { - try { - const association = buildAssociationName(type) - - for (const image of actor[association]) { - await image.destroy({ transaction: t }) - } - - actor[association] = [] - } catch (err) { - logger.error('Cannot remove old image of actor %s.', actor.url, { err }) - } - - return actor -} - -async function safeDeleteActorImage (actor: MActorImages, toDelete: MActorImage, type: ActorImageType, t: Transaction) { - try { - await toDelete.destroy({ transaction: t }) - - const association = buildAssociationName(type) - actor[association] = actor[association].filter(image => image.id !== toDelete.id) - } catch (err) { - logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) - } -} - -// --------------------------------------------------------------------------- - -export { - ImageInfo, - - updateActorImages, - deleteActorImages -} - -// --------------------------------------------------------------------------- - -function addActorImage (actor: MActorImages, type: ActorImageType, imageModel: MActorImage) { - const association = buildAssociationName(type) - if (!actor[association]) actor[association] = [] - - actor[association].push(imageModel) -} - -function buildAssociationName (type: ActorImageType) { - return type === ActorImageType.AVATAR - ? 'Avatars' - : 'Banners' -} diff --git a/server/lib/activitypub/actors/index.ts b/server/lib/activitypub/actors/index.ts deleted file mode 100644 index 5ee2a6f1a..000000000 --- a/server/lib/activitypub/actors/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './get' -export * from './image' -export * from './keys' -export * from './refresh' -export * from './updater' -export * from './webfinger' diff --git a/server/lib/activitypub/actors/keys.ts b/server/lib/activitypub/actors/keys.ts deleted file mode 100644 index c3d18abd8..000000000 --- a/server/lib/activitypub/actors/keys.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto' -import { MActor } from '@server/types/models' - -// Set account keys, this could be long so process after the account creation and do not block the client -async function generateAndSaveActorKeys (actor: T) { - const { publicKey, privateKey } = await createPrivateAndPublicKeys() - - actor.publicKey = publicKey - actor.privateKey = privateKey - - return actor.save() -} - -export { - generateAndSaveActorKeys -} diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts deleted file mode 100644 index d15cb5e90..000000000 --- a/server/lib/activitypub/actors/refresh.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CachePromiseFactory } from '@server/helpers/promise-cache' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { ActorLoadByUrlType } from '@server/lib/model-loaders' -import { ActorModel } from '@server/models/actor/actor' -import { MActorAccountChannelId, MActorFull } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { fetchRemoteActor } from './shared' -import { APActorUpdater } from './updater' -import { getUrlFromWebfinger } from './webfinger' - -type RefreshResult = Promise<{ actor: T | MActorFull, refreshed: boolean }> - -type RefreshOptions = { - actor: T - fetchedType: ActorLoadByUrlType -} - -const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions) => options.actor.url) - -function refreshActorIfNeeded (options: RefreshOptions): RefreshResult { - const actorArg = options.actor - if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false }) - - return promiseCache.run(options) -} - -export { - refreshActorIfNeeded -} - -// --------------------------------------------------------------------------- - -async function doRefresh (options: RefreshOptions): RefreshResult { - const { actor: actorArg, fetchedType } = options - - // We need more attributes - const actor = fetchedType === 'all' - ? actorArg as MActorFull - : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) - - const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url) - - logger.info('Refreshing actor %s.', actor.url, lTags()) - - try { - const actorUrl = await getActorUrl(actor) - const { actorObject } = await fetchRemoteActor(actorUrl) - - if (actorObject === undefined) { - logger.info('Cannot fetch remote actor %s in refresh actor.', actorUrl) - return { actor, refreshed: false } - } - - const updater = new APActorUpdater(actorObject, actor) - await updater.update() - - return { refreshed: true, actor } - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url, lTags()) - - actor.Account - ? await actor.Account.destroy() - : await actor.VideoChannel.destroy() - - return { actor: undefined, refreshed: false } - } - - logger.info('Cannot refresh actor %s.', actor.url, { err, ...lTags() }) - return { actor, refreshed: false } - } -} - -function getActorUrl (actor: MActorFull) { - return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) - .catch(err => { - logger.warn('Cannot get actor URL from webfinger, keeping the old one.', { err }) - return actor.url - }) -} diff --git a/server/lib/activitypub/actors/shared/creator.ts b/server/lib/activitypub/actors/shared/creator.ts deleted file mode 100644 index 500bc9912..000000000 --- a/server/lib/activitypub/actors/shared/creator.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { Op, Transaction } from 'sequelize' -import { sequelizeTypescript } from '@server/initializers/database' -import { AccountModel } from '@server/models/account/account' -import { ActorModel } from '@server/models/actor/actor' -import { ServerModel } from '@server/models/server/server' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MAccount, MAccountDefault, MActor, MActorFullActor, MActorId, MActorImages, MChannel, MServer } from '@server/types/models' -import { ActivityPubActor, ActorImageType } from '@shared/models' -import { updateActorImages } from '../image' -import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImagesInfoFromObject } from './object-to-model-attributes' -import { fetchActorFollowsCount } from './url-to-object' - -export class APActorCreator { - - constructor ( - private readonly actorObject: ActivityPubActor, - private readonly ownerActor?: MActorFullActor - ) { - - } - - async create (): Promise { - const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject) - - const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount)) - - return sequelizeTypescript.transaction(async t => { - const server = await this.setServer(actorInstance, t) - - const { actorCreated, created } = await this.saveActor(actorInstance, t) - - await this.setImageIfNeeded(actorCreated, ActorImageType.AVATAR, t) - await this.setImageIfNeeded(actorCreated, ActorImageType.BANNER, t) - - await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t) - - if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance - actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault - actorCreated.Account.Actor = actorCreated - } - - if (actorCreated.type === 'Group') { // Video channel - const channel = await this.saveVideoChannel(actorCreated, t) - actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account }) - } - - actorCreated.Server = server - - return actorCreated - }) - } - - private async setServer (actor: MActor, t: Transaction) { - const actorHost = new URL(actor.url).host - - const serverOptions = { - where: { - host: actorHost - }, - defaults: { - host: actorHost - }, - transaction: t - } - const [ server ] = await ServerModel.findOrCreate(serverOptions) - - // Save our new account in database - actor.serverId = server.id - - return server as MServer - } - - private async setImageIfNeeded (actor: MActor, type: ActorImageType, t: Transaction) { - const imagesInfo = getImagesInfoFromObject(this.actorObject, type) - if (imagesInfo.length === 0) return - - return updateActorImages(actor as MActorImages, type, imagesInfo, t) - } - - private async saveActor (actor: MActor, t: Transaction) { - // Force the actor creation using findOrCreate() instead of save() - // Sometimes Sequelize skips the save() when it thinks the instance already exists - // (which could be false in a retried query) - const [ actorCreated, created ] = await ActorModel.findOrCreate({ - defaults: actor.toJSON(), - where: { - [Op.or]: [ - { - url: actor.url - }, - { - serverId: actor.serverId, - preferredUsername: actor.preferredUsername - } - ] - }, - transaction: t - }) - - return { actorCreated, created } - } - - private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) { - // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards - if (created !== true && actorCreated.url !== newActor.url) { - // Only fix http://example.com/account/djidane to https://example.com/account/djidane - if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) { - throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`) - } - - actorCreated.url = newActor.url - await actorCreated.save({ transaction: t }) - } - } - - private async saveAccount (actor: MActorId, t: Transaction) { - const [ accountCreated ] = await AccountModel.findOrCreate({ - defaults: { - name: getActorDisplayNameFromObject(this.actorObject), - description: this.actorObject.summary, - actorId: actor.id - }, - where: { - actorId: actor.id - }, - transaction: t - }) - - return accountCreated as MAccount - } - - private async saveVideoChannel (actor: MActorId, t: Transaction) { - const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ - defaults: { - name: getActorDisplayNameFromObject(this.actorObject), - description: this.actorObject.summary, - support: this.actorObject.support, - actorId: actor.id, - accountId: this.ownerActor.Account.id - }, - where: { - actorId: actor.id - }, - transaction: t - }) - - return videoChannelCreated as MChannel - } -} diff --git a/server/lib/activitypub/actors/shared/index.ts b/server/lib/activitypub/actors/shared/index.ts deleted file mode 100644 index 52af1a8e1..000000000 --- a/server/lib/activitypub/actors/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './creator' -export * from './object-to-model-attributes' -export * from './url-to-object' diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts deleted file mode 100644 index 3ce332681..000000000 --- a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { MIMETYPES } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { FilteredModelAttributes } from '@server/types' -import { getLowercaseExtension } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { ActivityIconObject, ActivityPubActor, ActorImageType } from '@shared/models' - -function getActorAttributesFromObject ( - actorObject: ActivityPubActor, - followersCount: number, - followingCount: number -): FilteredModelAttributes { - return { - type: actorObject.type, - preferredUsername: actorObject.preferredUsername, - url: actorObject.id, - publicKey: actorObject.publicKey.publicKeyPem, - privateKey: null, - followersCount, - followingCount, - inboxUrl: actorObject.inbox, - outboxUrl: actorObject.outbox, - followersUrl: actorObject.followers, - followingUrl: actorObject.following, - - sharedInboxUrl: actorObject.endpoints?.sharedInbox - ? actorObject.endpoints.sharedInbox - : null - } -} - -function getImagesInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType) { - const iconsOrImages = type === ActorImageType.AVATAR - ? actorObject.icon - : actorObject.image - - return normalizeIconOrImage(iconsOrImages) - .map(iconOrImage => { - const mimetypes = MIMETYPES.IMAGE - - if (iconOrImage.type !== 'Image' || !isActivityPubUrlValid(iconOrImage.url)) return undefined - - let extension: string - - if (iconOrImage.mediaType) { - extension = mimetypes.MIMETYPE_EXT[iconOrImage.mediaType] - } else { - const tmp = getLowercaseExtension(iconOrImage.url) - - if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp - } - - if (!extension) return undefined - - return { - name: buildUUID() + extension, - fileUrl: iconOrImage.url, - height: iconOrImage.height, - width: iconOrImage.width, - type - } - }) - .filter(i => !!i) -} - -function getActorDisplayNameFromObject (actorObject: ActivityPubActor) { - return actorObject.name || actorObject.preferredUsername -} - -export { - getActorAttributesFromObject, - getImagesInfoFromObject, - getActorDisplayNameFromObject -} - -// --------------------------------------------------------------------------- - -function normalizeIconOrImage (icon: ActivityIconObject | ActivityIconObject[]): ActivityIconObject[] { - if (Array.isArray(icon)) return icon - if (icon) return [ icon ] - - return [] -} diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts deleted file mode 100644 index 73766bd50..000000000 --- a/server/lib/activitypub/actors/shared/url-to-object.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor' -import { logger } from '@server/helpers/logger' -import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models' -import { fetchAP } from '../../activity' -import { checkUrlsSameHost } from '../../url' - -async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> { - logger.info('Fetching remote actor %s.', actorUrl) - - const { body, statusCode } = await fetchAP(actorUrl) - - if (sanitizeAndCheckActorObject(body) === false) { - logger.debug('Remote actor JSON is not valid.', { actorJSON: body }) - return { actorObject: undefined, statusCode } - } - - if (checkUrlsSameHost(body.id, actorUrl) !== true) { - logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id) - return { actorObject: undefined, statusCode } - } - - return { - statusCode, - - actorObject: body - } -} - -async function fetchActorFollowsCount (actorObject: ActivityPubActor) { - let followersCount = 0 - let followingCount = 0 - - if (actorObject.followers) followersCount = await fetchActorTotalItems(actorObject.followers) - if (actorObject.following) followingCount = await fetchActorTotalItems(actorObject.following) - - return { followersCount, followingCount } -} - -// --------------------------------------------------------------------------- -export { - fetchActorFollowsCount, - fetchRemoteActor -} - -// --------------------------------------------------------------------------- - -async function fetchActorTotalItems (url: string) { - try { - const { body } = await fetchAP>(url) - - return body.totalItems || 0 - } catch (err) { - logger.info('Cannot fetch remote actor count %s.', url, { err }) - return 0 - } -} diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts deleted file mode 100644 index 5a92e7a22..000000000 --- a/server/lib/activitypub/actors/updater.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { AccountModel } from '@server/models/account/account' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' -import { ActivityPubActor, ActorImageType } from '@shared/models' -import { getOrCreateAPOwner } from './get' -import { updateActorImages } from './image' -import { fetchActorFollowsCount } from './shared' -import { getImagesInfoFromObject } from './shared/object-to-model-attributes' - -export class APActorUpdater { - - private readonly accountOrChannel: MAccount | MChannel - - constructor ( - private readonly actorObject: ActivityPubActor, - private readonly actor: MActorFull - ) { - if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel - else this.accountOrChannel = this.actor.Account - } - - async update () { - const avatarsInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.AVATAR) - const bannersInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.BANNER) - - try { - await this.updateActorInstance(this.actor, this.actorObject) - - this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername - this.accountOrChannel.description = this.actorObject.summary - - if (this.accountOrChannel instanceof VideoChannelModel) { - const owner = await getOrCreateAPOwner(this.actorObject, this.actorObject.url) - this.accountOrChannel.accountId = owner.Account.id - this.accountOrChannel.Account = owner.Account as AccountModel - - this.accountOrChannel.support = this.actorObject.support - } - - await runInReadCommittedTransaction(async t => { - await updateActorImages(this.actor, ActorImageType.BANNER, bannersInfo, t) - await updateActorImages(this.actor, ActorImageType.AVATAR, avatarsInfo, t) - }) - - await runInReadCommittedTransaction(async t => { - await this.actor.save({ transaction: t }) - await this.accountOrChannel.save({ transaction: t }) - }) - - logger.info('Remote account %s updated', this.actorObject.url) - } catch (err) { - if (this.actor !== undefined) { - await resetSequelizeInstance(this.actor) - } - - if (this.accountOrChannel !== undefined) { - await resetSequelizeInstance(this.accountOrChannel) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote account.', { err }) - throw err - } - } - - private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) { - const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject) - - actorInstance.type = actorObject.type - actorInstance.preferredUsername = actorObject.preferredUsername - actorInstance.url = actorObject.id - actorInstance.publicKey = actorObject.publicKey.publicKeyPem - actorInstance.followersCount = followersCount - actorInstance.followingCount = followingCount - actorInstance.inboxUrl = actorObject.inbox - actorInstance.outboxUrl = actorObject.outbox - actorInstance.followersUrl = actorObject.followers - actorInstance.followingUrl = actorObject.following - - if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published) - - if (actorObject.endpoints?.sharedInbox) { - actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox - } - - // Force actor update - actorInstance.changed('updatedAt', true) - } -} diff --git a/server/lib/activitypub/actors/webfinger.ts b/server/lib/activitypub/actors/webfinger.ts deleted file mode 100644 index b20a724da..000000000 --- a/server/lib/activitypub/actors/webfinger.ts +++ /dev/null @@ -1,67 +0,0 @@ -import WebFinger from 'webfinger.js' -import { isProdInstance } from '@server/helpers/core-utils' -import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { REQUEST_TIMEOUTS, WEBSERVER } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { MActorFull } from '@server/types/models' -import { WebFingerData } from '@shared/models' - -const webfinger = new WebFinger({ - webfist_fallback: false, - tls_only: isProdInstance(), - uri_fallback: false, - request_timeout: REQUEST_TIMEOUTS.DEFAULT -}) - -async function loadActorUrlOrGetFromWebfinger (uriArg: string) { - // Handle strings like @toto@example.com - const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg - - const [ name, host ] = uri.split('@') - let actor: MActorFull - - if (!host || host === WEBSERVER.HOST) { - actor = await ActorModel.loadLocalByName(name) - } else { - actor = await ActorModel.loadByNameAndHost(name, host) - } - - if (actor) return actor.url - - return getUrlFromWebfinger(uri) -} - -async function getUrlFromWebfinger (uri: string) { - const webfingerData: WebFingerData = await webfingerLookup(uri) - return getLinkOrThrow(webfingerData) -} - -// --------------------------------------------------------------------------- - -export { - getUrlFromWebfinger, - loadActorUrlOrGetFromWebfinger -} - -// --------------------------------------------------------------------------- - -function getLinkOrThrow (webfingerData: WebFingerData) { - if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.') - - const selfLink = webfingerData.links.find(l => l.rel === 'self') - if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) { - throw new Error('Cannot find self link or href is not a valid URL.') - } - - return selfLink.href -} - -function webfingerLookup (nameWithHost: string) { - return new Promise((res, rej) => { - webfinger.lookup(nameWithHost, (err, p) => { - if (err) return rej(err) - - return res(p.object) - }) - }) -} diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts deleted file mode 100644 index 6f5491387..000000000 --- a/server/lib/activitypub/audience.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { ActivityAudience } from '../../../shared/models/activitypub' -import { ACTIVITY_PUB } from '../../initializers/constants' -import { MActorFollowersUrl } from '../../types/models' - -function getAudience (actorSender: MActorFollowersUrl, isPublic = true) { - return buildAudience([ actorSender.followersUrl ], isPublic) -} - -function buildAudience (followerUrls: string[], isPublic = true) { - let to: string[] = [] - let cc: string[] = [] - - if (isPublic) { - to = [ ACTIVITY_PUB.PUBLIC ] - cc = followerUrls - } else { // Unlisted - to = [] - cc = [] - } - - return { to, cc } -} - -function audiencify (object: T, audience: ActivityAudience) { - return { ...audience, ...object } -} - -// --------------------------------------------------------------------------- - -export { - buildAudience, - getAudience, - audiencify -} diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts deleted file mode 100644 index c3acd7112..000000000 --- a/server/lib/activitypub/cache-file.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { Transaction } from 'sequelize' -import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' -import { CacheFileObject, VideoStreamingPlaylistType } from '@shared/models' -import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' - -async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { - const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) - - if (redundancyModel) { - return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) - } - - return createCacheFile(cacheFileObject, video, byActor, t) -} - -// --------------------------------------------------------------------------- - -export { - createOrUpdateCacheFile -} - -// --------------------------------------------------------------------------- - -function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { - const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) - - return VideoRedundancyModel.create(attributes, { transaction: t }) -} - -function updateCacheFile ( - cacheFileObject: CacheFileObject, - redundancyModel: MVideoRedundancy, - video: MVideoWithAllFiles, - byActor: MActorId, - t: Transaction -) { - if (redundancyModel.actorId !== byActor.id) { - throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') - } - - const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) - - redundancyModel.expiresOn = attributes.expiresOn - redundancyModel.fileUrl = attributes.fileUrl - - return redundancyModel.save({ transaction: t }) -} - -function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { - - if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { - const url = cacheFileObject.url - - const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) - if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) - - return { - expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, - url: cacheFileObject.id, - fileUrl: url.href, - strategy: null, - videoStreamingPlaylistId: playlist.id, - actorId: byActor.id - } - } - - const url = cacheFileObject.url - const videoFile = video.VideoFiles.find(f => { - return f.resolution === url.height && f.fps === url.fps - }) - - if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) - - return { - expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, - url: cacheFileObject.id, - fileUrl: url.href, - strategy: null, - videoFileId: videoFile.id, - actorId: byActor.id - } -} diff --git a/server/lib/activitypub/collection.ts b/server/lib/activitypub/collection.ts deleted file mode 100644 index a176cab51..000000000 --- a/server/lib/activitypub/collection.ts +++ /dev/null @@ -1,63 +0,0 @@ -import Bluebird from 'bluebird' -import validator from 'validator' -import { pageToStartAndCount } from '@server/helpers/core-utils' -import { ACTIVITY_PUB } from '@server/initializers/constants' -import { ResultList } from '@shared/models' -import { forceNumber } from '@shared/core-utils' - -type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird> | Promise> - -async function activityPubCollectionPagination ( - baseUrl: string, - handler: ActivityPubCollectionPaginationHandler, - page?: any, - size = ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE -) { - if (!page || !validator.isInt(page)) { - // We just display the first page URL, we only need the total items - const result = await handler(0, 1) - - return { - id: baseUrl, - type: 'OrderedCollection', - totalItems: result.total, - first: result.data.length === 0 - ? undefined - : baseUrl + '?page=1' - } - } - - const { start, count } = pageToStartAndCount(page, size) - const result = await handler(start, count) - - let next: string | undefined - let prev: string | undefined - - // Assert page is a number - page = forceNumber(page) - - // There are more results - if (result.total > page * size) { - next = baseUrl + '?page=' + (page + 1) - } - - if (page > 1) { - prev = baseUrl + '?page=' + (page - 1) - } - - return { - id: baseUrl + '?page=' + page, - type: 'OrderedCollectionPage', - prev, - next, - partOf: baseUrl, - orderedItems: result.data, - totalItems: result.total - } -} - -// --------------------------------------------------------------------------- - -export { - activityPubCollectionPagination -} diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts deleted file mode 100644 index 87eb498a3..000000000 --- a/server/lib/activitypub/context.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { ContextType } from '@shared/models' -import { Hooks } from '../plugins/hooks' - -async function activityPubContextify (data: T, type: ContextType) { - return { ...await getContextData(type), ...data } -} - -// --------------------------------------------------------------------------- - -export { - getContextData, - activityPubContextify -} - -// --------------------------------------------------------------------------- - -type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) } - -const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = { - Video: buildContext({ - Hashtag: 'as:Hashtag', - uuid: 'sc:identifier', - category: 'sc:category', - licence: 'sc:license', - subtitleLanguage: 'sc:subtitleLanguage', - sensitive: 'as:sensitive', - language: 'sc:inLanguage', - identifier: 'sc:identifier', - - isLiveBroadcast: 'sc:isLiveBroadcast', - liveSaveReplay: { - '@type': 'sc:Boolean', - '@id': 'pt:liveSaveReplay' - }, - permanentLive: { - '@type': 'sc:Boolean', - '@id': 'pt:permanentLive' - }, - latencyMode: { - '@type': 'sc:Number', - '@id': 'pt:latencyMode' - }, - - Infohash: 'pt:Infohash', - - tileWidth: { - '@type': 'sc:Number', - '@id': 'pt:tileWidth' - }, - tileHeight: { - '@type': 'sc:Number', - '@id': 'pt:tileHeight' - }, - tileDuration: { - '@type': 'sc:Number', - '@id': 'pt:tileDuration' - }, - - originallyPublishedAt: 'sc:datePublished', - - uploadDate: 'sc:uploadDate', - - views: { - '@type': 'sc:Number', - '@id': 'pt:views' - }, - state: { - '@type': 'sc:Number', - '@id': 'pt:state' - }, - size: { - '@type': 'sc:Number', - '@id': 'pt:size' - }, - fps: { - '@type': 'sc:Number', - '@id': 'pt:fps' - }, - commentsEnabled: { - '@type': 'sc:Boolean', - '@id': 'pt:commentsEnabled' - }, - downloadEnabled: { - '@type': 'sc:Boolean', - '@id': 'pt:downloadEnabled' - }, - waitTranscoding: { - '@type': 'sc:Boolean', - '@id': 'pt:waitTranscoding' - }, - support: { - '@type': 'sc:Text', - '@id': 'pt:support' - }, - likes: { - '@id': 'as:likes', - '@type': '@id' - }, - dislikes: { - '@id': 'as:dislikes', - '@type': '@id' - }, - shares: { - '@id': 'as:shares', - '@type': '@id' - }, - comments: { - '@id': 'as:comments', - '@type': '@id' - } - }), - - Playlist: buildContext({ - Playlist: 'pt:Playlist', - PlaylistElement: 'pt:PlaylistElement', - position: { - '@type': 'sc:Number', - '@id': 'pt:position' - }, - startTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:startTimestamp' - }, - stopTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:stopTimestamp' - }, - uuid: 'sc:identifier' - }), - - CacheFile: buildContext({ - expires: 'sc:expires', - CacheFile: 'pt:CacheFile' - }), - - Flag: buildContext({ - Hashtag: 'as:Hashtag' - }), - - Actor: buildContext({ - playlists: { - '@id': 'pt:playlists', - '@type': '@id' - }, - support: { - '@type': 'sc:Text', - '@id': 'pt:support' - }, - - // TODO: remove in a few versions, introduced in 4.2 - icons: 'as:icon' - }), - - WatchAction: buildContext({ - WatchAction: 'sc:WatchAction', - startTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:startTimestamp' - }, - stopTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:stopTimestamp' - }, - watchSection: { - '@type': 'sc:Number', - '@id': 'pt:stopTimestamp' - }, - uuid: 'sc:identifier' - }), - - Collection: buildContext(), - Follow: buildContext(), - Reject: buildContext(), - Accept: buildContext(), - View: buildContext(), - Announce: buildContext(), - Comment: buildContext(), - Delete: buildContext(), - Rate: buildContext() -} - -async function getContextData (type: ContextType) { - const contextData = await Hooks.wrapObject( - contextStore[type], - 'filter:activity-pub.activity.context.build.result' - ) - - return { '@context': contextData } -} - -function buildContext (contextValue?: ContextValue) { - const baseContext = [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - { - RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' - } - ] - - if (!contextValue) return baseContext - - return [ - ...baseContext, - - { - pt: 'https://joinpeertube.org/ns#', - sc: 'http://schema.org/', - - ...contextValue - } - ] -} diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts deleted file mode 100644 index b8348e8cf..000000000 --- a/server/lib/activitypub/crawl.ts +++ /dev/null @@ -1,58 +0,0 @@ -import Bluebird from 'bluebird' -import { URL } from 'url' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' -import { logger } from '../../helpers/logger' -import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants' -import { fetchAP } from './activity' - -type HandlerFunction = (items: T[]) => (Promise | Bluebird) -type CleanerFunction = (startedDate: Date) => Promise - -async function crawlCollectionPage (argUrl: string, handler: HandlerFunction, cleaner?: CleanerFunction) { - let url = argUrl - - logger.info('Crawling ActivityPub data on %s.', url) - - const startDate = new Date() - - const response = await fetchAP>(url) - const firstBody = response.body - - const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT - let i = 0 - let nextLink = firstBody.first - while (nextLink && i < limit) { - let body: any - - if (typeof nextLink === 'string') { - // Don't crawl ourselves - const remoteHost = new URL(nextLink).host - if (remoteHost === WEBSERVER.HOST) continue - - url = nextLink - - const res = await fetchAP>(url) - body = res.body - } else { - // nextLink is already the object we want - body = nextLink - } - - nextLink = body.next - i++ - - if (Array.isArray(body.orderedItems)) { - const items = body.orderedItems - logger.info('Processing %i ActivityPub items for %s.', items.length, url) - - await handler(items) - } - } - - if (cleaner) await retryTransactionWrapper(cleaner, startDate) -} - -export { - crawlCollectionPage -} diff --git a/server/lib/activitypub/follow.ts b/server/lib/activitypub/follow.ts deleted file mode 100644 index f6e2a48fd..000000000 --- a/server/lib/activitypub/follow.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SERVER_ACTOR_NAME } from '../../initializers/constants' -import { ServerModel } from '../../models/server/server' -import { MActorFollowActors } from '../../types/models' -import { JobQueue } from '../job-queue' - -async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors, transaction?: Transaction) { - if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return - - const follower = actorFollow.ActorFollower - - if (follower.type === 'Application' && follower.preferredUsername === SERVER_ACTOR_NAME) { - logger.info('Auto follow back %s.', follower.url) - - const me = await getServerActor() - - const server = await ServerModel.load(follower.serverId, transaction) - const host = server.host - - const payload = { - host, - name: SERVER_ACTOR_NAME, - followerActorId: me.id, - isAutoFollow: true - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) - } -} - -// If we only have an host, use a default account handle -function getRemoteNameAndHost (handleOrHost: string) { - let name = SERVER_ACTOR_NAME - let host = handleOrHost - - const splitted = handleOrHost.split('@') - if (splitted.length === 2) { - name = splitted[0] - host = splitted[1] - } - - return { name, host } -} - -export { - autoFollowBackIfNeeded, - getRemoteNameAndHost -} diff --git a/server/lib/activitypub/inbox-manager.ts b/server/lib/activitypub/inbox-manager.ts deleted file mode 100644 index 27778cc9d..000000000 --- a/server/lib/activitypub/inbox-manager.ts +++ /dev/null @@ -1,47 +0,0 @@ -import PQueue from 'p-queue' -import { logger } from '@server/helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' -import { MActorDefault, MActorSignature } from '@server/types/models' -import { Activity } from '@shared/models' -import { StatsManager } from '../stat-manager' -import { processActivities } from './process' - -class InboxManager { - - private static instance: InboxManager - private readonly inboxQueue: PQueue - - private constructor () { - this.inboxQueue = new PQueue({ concurrency: 1 }) - - setInterval(() => { - StatsManager.Instance.updateInboxWaiting(this.getActivityPubMessagesWaiting()) - }, SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS) - } - - addInboxMessage (param: { - activities: Activity[] - signatureActor?: MActorSignature - inboxActor?: MActorDefault - }) { - this.inboxQueue.add(() => { - const options = { signatureActor: param.signatureActor, inboxActor: param.inboxActor } - - return processActivities(param.activities, options) - }).catch(err => logger.error('Error with inbox queue.', { err })) - } - - getActivityPubMessagesWaiting () { - return this.inboxQueue.size + this.inboxQueue.pending - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - InboxManager -} diff --git a/server/lib/activitypub/local-video-viewer.ts b/server/lib/activitypub/local-video-viewer.ts deleted file mode 100644 index bdd746791..000000000 --- a/server/lib/activitypub/local-video-viewer.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Transaction } from 'sequelize' -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' -import { MVideo } from '@server/types/models' -import { WatchActionObject } from '@shared/models' -import { getDurationFromActivityStream } from './activity' - -async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) { - const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id) - if (stats) await stats.destroy({ transaction: t }) - - const localVideoViewer = await LocalVideoViewerModel.create({ - url: watchAction.id, - uuid: watchAction.uuid, - - watchTime: getDurationFromActivityStream(watchAction.duration), - - startDate: new Date(watchAction.startTime), - endDate: new Date(watchAction.endTime), - - country: watchAction.location - ? watchAction.location.addressCountry - : null, - - videoId: video.id - }, { transaction: t }) - - await LocalVideoViewerWatchSectionModel.bulkCreateSections({ - localVideoViewerId: localVideoViewer.id, - - watchSections: watchAction.watchSections.map(s => ({ - start: s.startTimestamp, - end: s.endTimestamp - })), - - transaction: t - }) -} - -// --------------------------------------------------------------------------- - -export { - createOrUpdateLocalVideoViewer -} diff --git a/server/lib/activitypub/outbox.ts b/server/lib/activitypub/outbox.ts deleted file mode 100644 index 5eef76871..000000000 --- a/server/lib/activitypub/outbox.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { ActorModel } from '@server/models/actor/actor' -import { getServerActor } from '@server/models/application/application' -import { JobQueue } from '../job-queue' - -async function addFetchOutboxJob (actor: Pick) { - // Don't fetch ourselves - const serverActor = await getServerActor() - if (serverActor.id === actor.id) { - logger.error('Cannot fetch our own outbox!') - return undefined - } - - const payload = { - uri: actor.outboxUrl, - type: 'activity' as 'activity' - } - - return JobQueue.Instance.createJobAsync({ type: 'activitypub-http-fetcher', payload }) -} - -export { - addFetchOutboxJob -} diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts deleted file mode 100644 index b24299f29..000000000 --- a/server/lib/activitypub/playlists/create-update.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { map } from 'bluebird' -import { isArray } from '@server/helpers/custom-validators/misc' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { updateRemotePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' -import { FilteredModelAttributes } from '@server/types' -import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models' -import { PlaylistObject } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { getAPId } from '../activity' -import { getOrCreateAPActor } from '../actors' -import { crawlCollectionPage } from '../crawl' -import { getOrCreateAPVideo } from '../videos' -import { - fetchRemotePlaylistElement, - fetchRemoteVideoPlaylist, - playlistElementObjectToDBAttributes, - playlistObjectToDBAttributes -} from './shared' - -const lTags = loggerTagsFactory('ap', 'video-playlist') - -async function createAccountPlaylists (playlistUrls: string[]) { - await map(playlistUrls, async playlistUrl => { - try { - const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) - if (exists === true) return - - const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) - - if (playlistObject === undefined) { - throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) - } - - return createOrUpdateVideoPlaylist(playlistObject) - } catch (err) { - logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) { - const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to) - - await setVideoChannel(playlistObject, playlistAttributes) - - const [ upsertPlaylist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) - - const playlistElementUrls = await fetchElementUrls(playlistObject) - - // Refetch playlist from DB since elements fetching could be long in time - const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null) - - await updatePlaylistThumbnail(playlistObject, playlist) - - const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist) - playlist.setVideosLength(elementsLength) - - return playlist -} - -// --------------------------------------------------------------------------- - -export { - createAccountPlaylists, - createOrUpdateVideoPlaylist -} - -// --------------------------------------------------------------------------- - -async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly) { - if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) { - throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) - } - - const actor = await getOrCreateAPActor(getAPId(playlistObject.attributedTo[0]), 'all') - - if (!actor.VideoChannel) { - logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) - return - } - - playlistAttributes.videoChannelId = actor.VideoChannel.id - playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id -} - -async function fetchElementUrls (playlistObject: PlaylistObject) { - let accItems: string[] = [] - await crawlCollectionPage(playlistObject.id, items => { - accItems = accItems.concat(items) - - return Promise.resolve() - }) - - return accItems -} - -async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) { - if (playlistObject.icon) { - let thumbnailModel: MThumbnail - - try { - thumbnailModel = await updateRemotePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) - await playlist.setAndSaveThumbnail(thumbnailModel, undefined) - } catch (err) { - logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) - - if (thumbnailModel) await thumbnailModel.removeThumbnail() - } - - return - } - - // Playlist does not have an icon, destroy existing one - if (playlist.hasThumbnail()) { - await playlist.Thumbnail.destroy() - playlist.Thumbnail = null - } -} - -async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { - const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist) - - await retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { - await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) - - for (const element of elementsToCreate) { - await VideoPlaylistElementModel.create(element, { transaction: t }) - } - })) - - logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) - - return elementsToCreate.length -} - -async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { - const elementsToCreate: FilteredModelAttributes[] = [] - - await map(elementUrls, async elementUrl => { - try { - const { elementObject } = await fetchRemotePlaylistElement(elementUrl) - - const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' }) - - elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video)) - } catch (err) { - logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) - - return elementsToCreate -} diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts deleted file mode 100644 index c34554d69..000000000 --- a/server/lib/activitypub/playlists/get.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { MVideoPlaylistFullSummary } from '@server/types/models' -import { APObjectId } from '@shared/models' -import { getAPId } from '../activity' -import { createOrUpdateVideoPlaylist } from './create-update' -import { scheduleRefreshIfNeeded } from './refresh' -import { fetchRemoteVideoPlaylist } from './shared' - -async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promise { - const playlistUrl = getAPId(playlistObjectArg) - - const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) - - if (playlistFromDatabase) { - scheduleRefreshIfNeeded(playlistFromDatabase) - - return playlistFromDatabase - } - - const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) - if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl) - - // playlistUrl is just an alias/redirection, so process object id instead - if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject) - - const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject) - - return playlistCreated -} - -// --------------------------------------------------------------------------- - -export { - getOrCreateAPVideoPlaylist -} diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts deleted file mode 100644 index e2470a674..000000000 --- a/server/lib/activitypub/playlists/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './get' -export * from './create-update' -export * from './refresh' diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts deleted file mode 100644 index 33260ea02..000000000 --- a/server/lib/activitypub/playlists/refresh.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { JobQueue } from '@server/lib/job-queue' -import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { createOrUpdateVideoPlaylist } from './create-update' -import { fetchRemoteVideoPlaylist } from './shared' - -function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) { - if (!playlist.isOutdated()) return - - JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } }) -} - -async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise { - if (!videoPlaylist.isOutdated()) return videoPlaylist - - const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url) - - logger.info('Refreshing playlist %s.', videoPlaylist.url, lTags()) - - try { - const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) - - if (playlistObject === undefined) { - logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags()) - - await videoPlaylist.setAsRefreshed() - return videoPlaylist - } - - await createOrUpdateVideoPlaylist(playlistObject) - - return videoPlaylist - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags()) - - await videoPlaylist.destroy() - return undefined - } - - logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() }) - - await videoPlaylist.setAsRefreshed() - return videoPlaylist - } -} - -export { - scheduleRefreshIfNeeded, - refreshVideoPlaylistIfNeeded -} diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts deleted file mode 100644 index a217f2291..000000000 --- a/server/lib/activitypub/playlists/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './object-to-model-attributes' -export * from './url-to-object' diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts deleted file mode 100644 index 753b5e660..000000000 --- a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { ACTIVITY_PUB } from '@server/initializers/constants' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' -import { MVideoId, MVideoPlaylistId } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models' - -function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) { - const privacy = to.includes(ACTIVITY_PUB.PUBLIC) - ? VideoPlaylistPrivacy.PUBLIC - : VideoPlaylistPrivacy.UNLISTED - - return { - name: playlistObject.name, - description: playlistObject.content, - privacy, - url: playlistObject.id, - uuid: playlistObject.uuid, - ownerAccountId: null, - videoChannelId: null, - createdAt: new Date(playlistObject.published), - updatedAt: new Date(playlistObject.updated) - } as AttributesOnly -} - -function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { - return { - position: elementObject.position, - url: elementObject.id, - startTimestamp: elementObject.startTimestamp || null, - stopTimestamp: elementObject.stopTimestamp || null, - videoPlaylistId: videoPlaylist.id, - videoId: video.id - } as AttributesOnly -} - -export { - playlistObjectToDBAttributes, - playlistElementObjectToDBAttributes -} diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts deleted file mode 100644 index fd9fe5558..000000000 --- a/server/lib/activitypub/playlists/shared/url-to-object.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist' -import { isArray } from '@server/helpers/custom-validators/misc' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { PlaylistElementObject, PlaylistObject } from '@shared/models' -import { fetchAP } from '../../activity' -import { checkUrlsSameHost } from '../../url' - -async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { - const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl) - - logger.info('Fetching remote playlist %s.', playlistUrl, lTags()) - - const { body, statusCode } = await fetchAP(playlistUrl) - - if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { - logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() }) - return { statusCode, playlistObject: undefined } - } - - if (!isArray(body.to)) { - logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() }) - return { statusCode, playlistObject: undefined } - } - - return { statusCode, playlistObject: body } -} - -async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> { - const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl) - - logger.debug('Fetching remote playlist element %s.', elementUrl, lTags()) - - const { body, statusCode } = await fetchAP(elementUrl) - - if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`) - - if (checkUrlsSameHost(body.id, elementUrl) !== true) { - throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) - } - - return { statusCode, elementObject: body } -} - -export { - fetchRemoteVideoPlaylist, - fetchRemotePlaylistElement -} diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts deleted file mode 100644 index 5466739c1..000000000 --- a/server/lib/activitypub/process/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './process' diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts deleted file mode 100644 index 077b01eda..000000000 --- a/server/lib/activitypub/process/process-accept.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { ActivityAccept } from '../../../../shared/models/activitypub' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorDefault, MActorSignature } from '../../../types/models' -import { addFetchOutboxJob } from '../outbox' - -async function processAcceptActivity (options: APProcessorOptions) { - const { byActor: targetActor, inboxActor } = options - if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') - - return processAccept(inboxActor, targetActor) -} - -// --------------------------------------------------------------------------- - -export { - processAcceptActivity -} - -// --------------------------------------------------------------------------- - -async function processAccept (actor: MActorDefault, targetActor: MActorSignature) { - const follow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id) - if (!follow) throw new Error('Cannot find associated follow.') - - if (follow.state !== 'accepted') { - follow.state = 'accepted' - await follow.save() - - await addFetchOutboxJob(targetActor) - } -} diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts deleted file mode 100644 index 9cc87ee27..000000000 --- a/server/lib/activitypub/process/process-announce.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { getAPId } from '@server/lib/activitypub/activity' -import { ActivityAnnounce } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { VideoShareModel } from '../../../models/video/video-share' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' -import { Notifier } from '../../notifier' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { getOrCreateAPVideo } from '../videos' - -async function processAnnounceActivity (options: APProcessorOptions) { - const { activity, byActor: actorAnnouncer } = options - // Only notify if it is not from a fetcher job - const notify = options.fromFetch !== true - - // Announces on accounts are not supported - if (actorAnnouncer.type !== 'Application' && actorAnnouncer.type !== 'Group') return - - return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity, notify) -} - -// --------------------------------------------------------------------------- - -export { - processAnnounceActivity -} - -// --------------------------------------------------------------------------- - -async function processVideoShare (actorAnnouncer: MActorSignature, activity: ActivityAnnounce, notify: boolean) { - const objectUri = getAPId(activity.object) - - let video: MVideoAccountLightBlacklistAllFiles - let videoCreated: boolean - - try { - const result = await getOrCreateAPVideo({ videoObject: objectUri }) - video = result.video - videoCreated = result.created - } catch (err) { - logger.debug('Cannot process share of %s. Maybe this is not a video object, so just skipping.', objectUri, { err }) - return - } - - await sequelizeTypescript.transaction(async t => { - // Add share entry - - const share = { - actorId: actorAnnouncer.id, - videoId: video.id, - url: activity.id - } - - const [ , created ] = await VideoShareModel.findOrCreate({ - where: { - url: activity.id - }, - defaults: share, - transaction: t - }) - - if (video.isOwned() && created === true) { - // Don't resend the activity to the sender - const exceptions = [ actorAnnouncer ] - - await forwardVideoRelatedActivity(activity, t, exceptions, video) - } - - return undefined - }) - - if (videoCreated && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) -} diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts deleted file mode 100644 index 5f980de65..000000000 --- a/server/lib/activitypub/process/process-create.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { isRedundancyAccepted } from '@server/lib/redundancy' -import { VideoModel } from '@server/models/video/video' -import { - AbuseObject, - ActivityCreate, - ActivityCreateObject, - ActivityObject, - CacheFileObject, - PlaylistObject, - VideoCommentObject, - VideoObject, - WatchActionObject -} from '@shared/models' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' -import { Notifier } from '../../notifier' -import { fetchAPObjectIfNeeded } from '../activity' -import { createOrUpdateCacheFile } from '../cache-file' -import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' -import { createOrUpdateVideoPlaylist } from '../playlists' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { resolveThread } from '../video-comments' -import { getOrCreateAPVideo } from '../videos' - -async function processCreateActivity (options: APProcessorOptions>) { - const { activity, byActor } = options - - // Only notify if it is not from a fetcher job - const notify = options.fromFetch !== true - const activityObject = await fetchAPObjectIfNeeded>(activity.object) - const activityType = activityObject.type - - if (activityType === 'Video') { - return processCreateVideo(activityObject, notify) - } - - if (activityType === 'Note') { - // Comments will be fetched from videos - if (options.fromFetch) return - - return retryTransactionWrapper(processCreateVideoComment, activity, activityObject, byActor, notify) - } - - if (activityType === 'WatchAction') { - return retryTransactionWrapper(processCreateWatchAction, activityObject) - } - - if (activityType === 'CacheFile') { - return retryTransactionWrapper(processCreateCacheFile, activity, activityObject, byActor) - } - - if (activityType === 'Playlist') { - return retryTransactionWrapper(processCreatePlaylist, activity, activityObject, byActor) - } - - logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) - return Promise.resolve(undefined) -} - -// --------------------------------------------------------------------------- - -export { - processCreateActivity -} - -// --------------------------------------------------------------------------- - -async function processCreateVideo (videoToCreateData: VideoObject, notify: boolean) { - const syncParam = { rates: false, shares: false, comments: false, refreshVideo: false } - const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) - - if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) - - return video -} - -async function processCreateCacheFile ( - activity: ActivityCreate, - cacheFile: CacheFileObject, - byActor: MActorSignature -) { - if (await isRedundancyAccepted(activity, byActor) !== true) return - - const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) - - await sequelizeTypescript.transaction(async t => { - return createOrUpdateCacheFile(cacheFile, video, byActor, t) - }) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} - -async function processCreateWatchAction (watchAction: WatchActionObject) { - if (watchAction.actionStatus !== 'CompletedActionStatus') return - - const video = await VideoModel.loadByUrl(watchAction.object) - if (video.remote) return - - await sequelizeTypescript.transaction(async t => { - return createOrUpdateLocalVideoViewer(watchAction, video, t) - }) -} - -async function processCreateVideoComment ( - activity: ActivityCreate, - commentObject: VideoCommentObject, - byActor: MActorSignature, - notify: boolean -) { - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) - - let video: MVideoAccountLightBlacklistAllFiles - let created: boolean - let comment: MCommentOwnerVideo - - try { - const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) - if (!resolveThreadResult) return // Comment not accepted - - video = resolveThreadResult.video - created = resolveThreadResult.commentCreated - comment = resolveThreadResult.comment - } catch (err) { - logger.debug( - 'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.', - commentObject.inReplyTo, - { err } - ) - return - } - - // Try to not forward unwanted comments on our videos - if (video.isOwned()) { - if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) { - logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url) - return - } - - if (created === true) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } - } - - if (created && notify) Notifier.Instance.notifyOnNewComment(comment) -} - -async function processCreatePlaylist ( - activity: ActivityCreate, - playlistObject: PlaylistObject, - byActor: MActorSignature -) { - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) - - await createOrUpdateVideoPlaylist(playlistObject, activity.to) -} diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts deleted file mode 100644 index ac0e7e235..000000000 --- a/server/lib/activitypub/process/process-delete.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { ActivityDelete } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorModel } from '../../../models/actor/actor' -import { VideoModel } from '../../../models/video/video' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoPlaylistModel } from '../../../models/video/video-playlist' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { - MAccountActor, - MActor, - MActorFull, - MActorSignature, - MChannelAccountActor, - MChannelActor, - MCommentOwnerVideo -} from '../../../types/models' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' - -async function processDeleteActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id - - if (activity.actor === objectUrl) { - // We need more attributes (all the account and channel) - const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) - - if (byActorFull.type === 'Person') { - if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.') - - const accountToDelete = byActorFull.Account as MAccountActor - accountToDelete.Actor = byActorFull - - return retryTransactionWrapper(processDeleteAccount, accountToDelete) - } else if (byActorFull.type === 'Group') { - if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.') - - const channelToDelete = byActorFull.VideoChannel as MChannelAccountActor & { Actor: MActorFull } - channelToDelete.Actor = byActorFull - return retryTransactionWrapper(processDeleteVideoChannel, channelToDelete) - } - } - - { - const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(objectUrl) - if (videoCommentInstance) { - return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity) - } - } - - { - const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) - if (videoInstance) { - if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`) - - return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance) - } - } - - { - const videoPlaylist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(objectUrl) - if (videoPlaylist) { - if (videoPlaylist.isOwned()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`) - - return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist) - } - } - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - processDeleteActivity -} - -// --------------------------------------------------------------------------- - -async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) { - logger.debug('Removing remote video "%s".', videoToDelete.uuid) - - await sequelizeTypescript.transaction(async t => { - if (videoToDelete.VideoChannel.Account.Actor.id !== actor.id) { - throw new Error('Account ' + actor.url + ' does not own video channel ' + videoToDelete.VideoChannel.Actor.url) - } - - await videoToDelete.destroy({ transaction: t }) - }) - - logger.info('Remote video with uuid %s removed.', videoToDelete.uuid) -} - -async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: VideoPlaylistModel) { - logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid) - - await sequelizeTypescript.transaction(async t => { - if (playlistToDelete.OwnerAccount.Actor.id !== actor.id) { - throw new Error('Account ' + actor.url + ' does not own video playlist ' + playlistToDelete.url) - } - - await playlistToDelete.destroy({ transaction: t }) - }) - - logger.info('Remote video playlist with uuid %s removed.', playlistToDelete.uuid) -} - -async function processDeleteAccount (accountToRemove: MAccountActor) { - logger.debug('Removing remote account "%s".', accountToRemove.Actor.url) - - await sequelizeTypescript.transaction(async t => { - await accountToRemove.destroy({ transaction: t }) - }) - - logger.info('Remote account %s removed.', accountToRemove.Actor.url) -} - -async function processDeleteVideoChannel (videoChannelToRemove: MChannelActor) { - logger.debug('Removing remote video channel "%s".', videoChannelToRemove.Actor.url) - - await sequelizeTypescript.transaction(async t => { - await videoChannelToRemove.destroy({ transaction: t }) - }) - - logger.info('Remote video channel %s removed.', videoChannelToRemove.Actor.url) -} - -function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { - // Already deleted - if (videoComment.isDeleted()) return Promise.resolve() - - logger.debug('Removing remote video comment "%s".', videoComment.url) - - return sequelizeTypescript.transaction(async t => { - if (byActor.Account.id !== videoComment.Account.id && byActor.Account.id !== videoComment.Video.VideoChannel.accountId) { - throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) - } - - videoComment.markAsDeleted() - - await videoComment.save({ transaction: t }) - - if (videoComment.Video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, t, exceptions, videoComment.Video) - } - - logger.info('Remote video comment %s removed.', videoComment.url) - }) -} diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts deleted file mode 100644 index 4e270f917..000000000 --- a/server/lib/activitypub/process/process-dislike.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { ActivityDislike } from '@shared/models' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' - -async function processDislikeActivity (options: APProcessorOptions) { - const { activity, byActor } = options - return retryTransactionWrapper(processDislike, activity, byActor) -} - -// --------------------------------------------------------------------------- - -export { - processDislikeActivity -} - -// --------------------------------------------------------------------------- - -async function processDislike (activity: ActivityDislike, byActor: MActorSignature) { - const dislikeObject = activity.object - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeObject, fetchType: 'only-video' }) - - // We don't care about dislikes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(onlyVideo.id, t) - - const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) - if (existingRate && existingRate.type === 'dislike') return - - await video.increment('dislikes', { transaction: t }) - video.dislikes++ - - if (existingRate && existingRate.type === 'like') { - await video.decrement('likes', { transaction: t }) - video.likes-- - } - - const rate = existingRate || new AccountVideoRateModel() - rate.type = 'dislike' - rate.videoId = video.id - rate.accountId = byAccount.id - rate.url = activity.id - - await rate.save({ transaction: t }) - - await federateVideoIfNeeded(video, false, t) - }) -} diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts deleted file mode 100644 index bea285670..000000000 --- a/server/lib/activitypub/process/process-flag.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation' -import { AccountModel } from '@server/models/account/account' -import { VideoModel } from '@server/models/video/video' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' -import { AbuseState, ActivityFlag } from '@shared/models' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { getAPId } from '../../../lib/activitypub/activity' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../types/models' - -async function processFlagActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - return retryTransactionWrapper(processCreateAbuse, activity, byActor) -} - -// --------------------------------------------------------------------------- - -export { - processFlagActivity -} - -// --------------------------------------------------------------------------- - -async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature) { - const account = byActor.Account - if (!account) throw new Error('Cannot create abuse with the non account actor ' + byActor.url) - - const reporterAccount = await AccountModel.load(account.id) - - const objects = Array.isArray(flag.object) ? flag.object : [ flag.object ] - - const tags = Array.isArray(flag.tag) ? flag.tag : [] - const predefinedReasons = tags.map(tag => abusePredefinedReasonsMap[tag.name]) - .filter(v => !isNaN(v)) - - const startAt = flag.startAt - const endAt = flag.endAt - - for (const object of objects) { - try { - const uri = getAPId(object) - - logger.debug('Reporting remote abuse for object %s.', uri) - - await sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(uri, t) - let videoComment: MCommentOwnerVideo - let flaggedAccount: MAccountDefault - - if (!video) videoComment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(uri, t) - if (!videoComment) flaggedAccount = await AccountModel.loadByUrl(uri, t) - - if (!video && !videoComment && !flaggedAccount) { - logger.warn('Cannot flag unknown entity %s.', object) - return - } - - const baseAbuse = { - reporterAccountId: reporterAccount.id, - reason: flag.content, - state: AbuseState.PENDING, - predefinedReasons - } - - if (video) { - return createVideoAbuse({ - baseAbuse, - startAt, - endAt, - reporterAccount, - transaction: t, - videoInstance: video, - skipNotification: false - }) - } - - if (videoComment) { - return createVideoCommentAbuse({ - baseAbuse, - reporterAccount, - transaction: t, - commentInstance: videoComment, - skipNotification: false - }) - } - - return await createAccountAbuse({ - baseAbuse, - reporterAccount, - transaction: t, - accountInstance: flaggedAccount, - skipNotification: false - }) - }) - } catch (err) { - logger.debug('Cannot process report of %s', getAPId(object), { err }) - } - } -} diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts deleted file mode 100644 index 7def753d5..000000000 --- a/server/lib/activitypub/process/process-follow.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { AccountModel } from '@server/models/account/account' -import { getServerActor } from '@server/models/application/application' -import { ActivityFollow } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { CONFIG } from '../../../initializers/config' -import { sequelizeTypescript } from '../../../initializers/database' -import { getAPId } from '../../../lib/activitypub/activity' -import { ActorModel } from '../../../models/actor/actor' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorFollow, MActorFull, MActorId, MActorSignature } from '../../../types/models' -import { Notifier } from '../../notifier' -import { autoFollowBackIfNeeded } from '../follow' -import { sendAccept, sendReject } from '../send' - -async function processFollowActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - const activityId = activity.id - const objectId = getAPId(activity.object) - - return retryTransactionWrapper(processFollow, byActor, activityId, objectId) -} - -// --------------------------------------------------------------------------- - -export { - processFollowActivity -} - -// --------------------------------------------------------------------------- - -async function processFollow (byActor: MActorSignature, activityId: string, targetActorURL: string) { - const { actorFollow, created, targetActor } = await sequelizeTypescript.transaction(async t => { - const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) - - if (!targetActor) throw new Error('Unknown actor') - if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') - - if (await rejectIfInstanceFollowDisabled(byActor, activityId, targetActor)) return { actorFollow: undefined } - if (await rejectIfMuted(byActor, activityId, targetActor)) return { actorFollow: undefined } - - const [ actorFollow, created ] = await ActorFollowModel.findOrCreateCustom({ - byActor, - targetActor, - activityId, - state: await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL - ? 'pending' - : 'accepted', - transaction: t - }) - - if (rejectIfAlreadyRejected(actorFollow, byActor, activityId, targetActor)) return { actorFollow: undefined } - - await acceptIfNeeded(actorFollow, targetActor, t) - - await fixFollowURLIfNeeded(actorFollow, activityId, t) - - actorFollow.ActorFollower = byActor - actorFollow.ActorFollowing = targetActor - - // Target sends to actor he accepted the follow request - if (actorFollow.state === 'accepted') { - sendAccept(actorFollow) - - await autoFollowBackIfNeeded(actorFollow, t) - } - - return { actorFollow, created, targetActor } - }) - - // Rejected - if (!actorFollow) return - - if (created) { - const follower = await ActorModel.loadFull(byActor.id) - const actorFollowFull = Object.assign(actorFollow, { ActorFollowing: targetActor, ActorFollower: follower }) - - if (await isFollowingInstance(targetActor)) { - Notifier.Instance.notifyOfNewInstanceFollow(actorFollowFull) - } else { - Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) - } - } - - logger.info('Actor %s is followed by actor %s.', targetActorURL, byActor.url) -} - -async function rejectIfInstanceFollowDisabled (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { - if (await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { - logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) - - sendReject(activityId, byActor, targetActor) - - return true - } - - return false -} - -async function rejectIfMuted (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { - const followerAccount = await AccountModel.load(byActor.Account.id) - const followingAccountId = targetActor.Account - - if (followerAccount && await isBlockedByServerOrAccount(followerAccount, followingAccountId)) { - logger.info('Rejecting %s because follower is muted.', byActor.url) - - sendReject(activityId, byActor, targetActor) - - return true - } - - return false -} - -function rejectIfAlreadyRejected (actorFollow: MActorFollow, byActor: MActorSignature, activityId: string, targetActor: MActorFull) { - // Already rejected - if (actorFollow.state === 'rejected') { - logger.info('Rejecting %s because follow is already rejected.', byActor.url) - - sendReject(activityId, byActor, targetActor) - - return true - } - - return false -} - -async function acceptIfNeeded (actorFollow: MActorFollow, targetActor: MActorFull, transaction: Transaction) { - // Set the follow as accepted if the remote actor follows a channel or account - // Or if the instance automatically accepts followers - if (actorFollow.state === 'accepted') return - if (!await isFollowingInstance(targetActor)) return - if (CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL === true && await isFollowingInstance(targetActor)) return - - actorFollow.state = 'accepted' - - await actorFollow.save({ transaction }) -} - -async function fixFollowURLIfNeeded (actorFollow: MActorFollow, activityId: string, transaction: Transaction) { - // Before PeerTube V3 we did not save the follow ID. Try to fix these old follows - if (!actorFollow.url) { - actorFollow.url = activityId - await actorFollow.save({ transaction }) - } -} - -async function isFollowingInstance (targetActor: MActorId) { - const serverActor = await getServerActor() - - return targetActor.id === serverActor.id -} diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts deleted file mode 100644 index 580a05bcd..000000000 --- a/server/lib/activitypub/process/process-like.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { ActivityLike } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { getAPId } from '../../../lib/activitypub/activity' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' - -async function processLikeActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - return retryTransactionWrapper(processLikeVideo, byActor, activity) -} - -// --------------------------------------------------------------------------- - -export { - processLikeActivity -} - -// --------------------------------------------------------------------------- - -async function processLikeVideo (byActor: MActorSignature, activity: ActivityLike) { - const videoUrl = getAPId(activity.object) - - const byAccount = byActor.Account - if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: videoUrl, fetchType: 'only-video' }) - - // We don't care about likes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(onlyVideo.id, t) - - const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) - if (existingRate && existingRate.type === 'like') return - - if (existingRate && existingRate.type === 'dislike') { - await video.decrement('dislikes', { transaction: t }) - video.dislikes-- - } - - await video.increment('likes', { transaction: t }) - video.likes++ - - const rate = existingRate || new AccountVideoRateModel() - rate.type = 'like' - rate.videoId = video.id - rate.accountId = byAccount.id - rate.url = activity.id - - await rate.save({ transaction: t }) - - await federateVideoIfNeeded(video, false, t) - }) -} diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts deleted file mode 100644 index db7ff24d8..000000000 --- a/server/lib/activitypub/process/process-reject.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { ActivityReject } from '../../../../shared/models/activitypub/activity' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActor } from '../../../types/models' - -async function processRejectActivity (options: APProcessorOptions) { - const { byActor: targetActor, inboxActor } = options - if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') - - return processReject(inboxActor, targetActor) -} - -// --------------------------------------------------------------------------- - -export { - processRejectActivity -} - -// --------------------------------------------------------------------------- - -async function processReject (follower: MActor, targetActor: MActor) { - return sequelizeTypescript.transaction(async t => { - const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t) - - if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`) - - actorFollow.state = 'rejected' - await actorFollow.save({ transaction: t }) - - return undefined - }) -} diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts deleted file mode 100644 index a9d8199de..000000000 --- a/server/lib/activitypub/process/process-undo.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { - ActivityAnnounce, - ActivityCreate, - ActivityDislike, - ActivityFollow, - ActivityLike, - ActivityUndo, - ActivityUndoObject, - CacheFileObject -} from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { ActorModel } from '../../../models/actor/actor' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' -import { VideoShareModel } from '../../../models/video/video-share' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { fetchAPObjectIfNeeded } from '../activity' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' - -async function processUndoActivity (options: APProcessorOptions>) { - const { activity, byActor } = options - const activityToUndo = activity.object - - if (activityToUndo.type === 'Like') { - return retryTransactionWrapper(processUndoLike, byActor, activity) - } - - if (activityToUndo.type === 'Create') { - const objectToUndo = await fetchAPObjectIfNeeded(activityToUndo.object) - - if (objectToUndo.type === 'CacheFile') { - return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo) - } - } - - if (activityToUndo.type === 'Dislike') { - return retryTransactionWrapper(processUndoDislike, byActor, activity) - } - - if (activityToUndo.type === 'Follow') { - return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) - } - - if (activityToUndo.type === 'Announce') { - return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo) - } - - logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - processUndoActivity -} - -// --------------------------------------------------------------------------- - -async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { - const likeActivity = activity.object - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) - // We don't care about likes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - - const video = await VideoModel.loadFull(onlyVideo.id, t) - const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, likeActivity.id, t) - if (!rate || rate.type !== 'like') { - logger.warn('Unknown like by account %d for video %d.', byActor.Account.id, video.id) - return - } - - await rate.destroy({ transaction: t }) - await video.decrement('likes', { transaction: t }) - - video.likes-- - await federateVideoIfNeeded(video, false, t) - }) -} - -async function processUndoDislike (byActor: MActorSignature, activity: ActivityUndo) { - const dislikeActivity = activity.object - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeActivity.object }) - // We don't care about likes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - - const video = await VideoModel.loadFull(onlyVideo.id, t) - const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, dislikeActivity.id, t) - if (!rate || rate.type !== 'dislike') { - logger.warn(`Unknown dislike by account %d for video %d.`, byActor.Account.id, video.id) - return - } - - await rate.destroy({ transaction: t }) - await video.decrement('dislikes', { transaction: t }) - video.dislikes-- - - await federateVideoIfNeeded(video, false, t) - }) -} - -// --------------------------------------------------------------------------- - -async function processUndoCacheFile ( - byActor: MActorSignature, - activity: ActivityUndo>, - cacheFileObject: CacheFileObject -) { - const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) - - return sequelizeTypescript.transaction(async t => { - const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) - if (!cacheFile) { - logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) - return - } - - if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') - - await cacheFile.destroy({ transaction: t }) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, t, exceptions, video) - } - }) -} - -function processUndoAnnounce (byActor: MActorSignature, announceActivity: ActivityAnnounce) { - return sequelizeTypescript.transaction(async t => { - const share = await VideoShareModel.loadByUrl(announceActivity.id, t) - if (!share) { - logger.warn('Unknown video share %d', announceActivity.id) - return - } - - if (share.actorId !== byActor.id) throw new Error(`${share.url} is not shared by ${byActor.url}.`) - - await share.destroy({ transaction: t }) - - if (share.Video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(announceActivity, t, exceptions, share.Video) - } - }) -} - -// --------------------------------------------------------------------------- - -function processUndoFollow (follower: MActorSignature, followActivity: ActivityFollow) { - return sequelizeTypescript.transaction(async t => { - const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t) - const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) - - if (!actorFollow) { - logger.warn('Unknown actor follow %d -> %d.', follower.id, following.id) - return - } - - await actorFollow.destroy({ transaction: t }) - - return undefined - }) -} diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts deleted file mode 100644 index 304ed9de6..000000000 --- a/server/lib/activitypub/process/process-update.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { isRedundancyAccepted } from '@server/lib/redundancy' -import { ActivityUpdate, ActivityUpdateObject, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' -import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' -import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' -import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' -import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorModel } from '../../../models/actor/actor' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorFull, MActorSignature } from '../../../types/models' -import { fetchAPObjectIfNeeded } from '../activity' -import { APActorUpdater } from '../actors/updater' -import { createOrUpdateCacheFile } from '../cache-file' -import { createOrUpdateVideoPlaylist } from '../playlists' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { APVideoUpdater, getOrCreateAPVideo } from '../videos' - -async function processUpdateActivity (options: APProcessorOptions>) { - const { activity, byActor } = options - - const object = await fetchAPObjectIfNeeded(activity.object) - const objectType = object.type - - if (objectType === 'Video') { - return retryTransactionWrapper(processUpdateVideo, activity) - } - - if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { - // We need more attributes - const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) - return retryTransactionWrapper(processUpdateActor, byActorFull, object) - } - - if (objectType === 'CacheFile') { - // We need more attributes - const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) - return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity, object) - } - - if (objectType === 'Playlist') { - return retryTransactionWrapper(processUpdatePlaylist, byActor, activity, object) - } - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - processUpdateActivity -} - -// --------------------------------------------------------------------------- - -async function processUpdateVideo (activity: ActivityUpdate) { - const videoObject = activity.object as VideoObject - - if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { - logger.debug('Video sent by update is not valid.', { videoObject }) - return undefined - } - - const { video, created } = await getOrCreateAPVideo({ - videoObject: videoObject.id, - allowRefresh: false, - fetchType: 'all' - }) - // We did not have this video, it has been created so no need to update - if (created) return - - const updater = new APVideoUpdater(videoObject, video) - return updater.update(activity.to) -} - -async function processUpdateCacheFile ( - byActor: MActorSignature, - activity: ActivityUpdate, - cacheFileObject: CacheFileObject -) { - if (await isRedundancyAccepted(activity, byActor) !== true) return - - if (!isCacheFileObjectValid(cacheFileObject)) { - logger.debug('Cache file object sent by update is not valid.', { cacheFileObject }) - return undefined - } - - const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) - - await sequelizeTypescript.transaction(async t => { - await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) - }) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} - -async function processUpdateActor (actor: MActorFull, actorObject: ActivityPubActor) { - logger.debug('Updating remote account "%s".', actorObject.url) - - const updater = new APActorUpdater(actorObject, actor) - return updater.update() -} - -async function processUpdatePlaylist ( - byActor: MActorSignature, - activity: ActivityUpdate, - playlistObject: PlaylistObject -) { - const byAccount = byActor.Account - if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) - - await createOrUpdateVideoPlaylist(playlistObject, activity.to) -} diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts deleted file mode 100644 index e49506d82..000000000 --- a/server/lib/activitypub/process/process-view.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { ActivityView } from '../../../../shared/models/activitypub' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { getOrCreateAPVideo } from '../videos' - -async function processViewActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - return processCreateView(activity, byActor) -} - -// --------------------------------------------------------------------------- - -export { - processViewActivity -} - -// --------------------------------------------------------------------------- - -async function processCreateView (activity: ActivityView, byActor: MActorSignature) { - const videoObject = activity.object - - const { video } = await getOrCreateAPVideo({ - videoObject, - fetchType: 'only-video', - allowRefresh: false - }) - - const viewerExpires = activity.expires - ? new Date(activity.expires) - : undefined - - await VideoViewsManager.Instance.processRemoteView({ video, viewerId: activity.id, viewerExpires }) - - if (video.isOwned()) { - // Forward the view but don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts deleted file mode 100644 index 2bc3dce03..000000000 --- a/server/lib/activitypub/process/process.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { StatsManager } from '@server/lib/stat-manager' -import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { logger } from '../../../helpers/logger' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorDefault, MActorSignature } from '../../../types/models' -import { getAPId } from '../activity' -import { getOrCreateAPActor } from '../actors' -import { checkUrlsSameHost } from '../url' -import { processAcceptActivity } from './process-accept' -import { processAnnounceActivity } from './process-announce' -import { processCreateActivity } from './process-create' -import { processDeleteActivity } from './process-delete' -import { processDislikeActivity } from './process-dislike' -import { processFlagActivity } from './process-flag' -import { processFollowActivity } from './process-follow' -import { processLikeActivity } from './process-like' -import { processRejectActivity } from './process-reject' -import { processUndoActivity } from './process-undo' -import { processUpdateActivity } from './process-update' -import { processViewActivity } from './process-view' - -const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions) => Promise } = { - Create: processCreateActivity, - Update: processUpdateActivity, - Delete: processDeleteActivity, - Follow: processFollowActivity, - Accept: processAcceptActivity, - Reject: processRejectActivity, - Announce: processAnnounceActivity, - Undo: processUndoActivity, - Like: processLikeActivity, - Dislike: processDislikeActivity, - Flag: processFlagActivity, - View: processViewActivity -} - -async function processActivities ( - activities: Activity[], - options: { - signatureActor?: MActorSignature - inboxActor?: MActorDefault - outboxUrl?: string - fromFetch?: boolean - } = {} -) { - const { outboxUrl, signatureActor, inboxActor, fromFetch = false } = options - - const actorsCache: { [ url: string ]: MActorSignature } = {} - - for (const activity of activities) { - if (!signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { - logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) - continue - } - - const actorUrl = getAPId(activity.actor) - - // When we fetch remote data, we don't have signature - if (signatureActor && actorUrl !== signatureActor.url) { - logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, signatureActor.url) - continue - } - - if (outboxUrl && checkUrlsSameHost(outboxUrl, actorUrl) !== true) { - logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', outboxUrl, actorUrl) - continue - } - - const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl) - actorsCache[actorUrl] = byActor - - const activityProcessor = processActivity[activity.type] - if (activityProcessor === undefined) { - logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) - continue - } - - try { - await activityProcessor({ activity, byActor, inboxActor, fromFetch }) - - StatsManager.Instance.addInboxProcessedSuccess(activity.type) - } catch (err) { - logger.warn('Cannot process activity %s.', activity.type, { err }) - - StatsManager.Instance.addInboxProcessedError(activity.type) - } - } -} - -export { - processActivities -} diff --git a/server/lib/activitypub/send/http.ts b/server/lib/activitypub/send/http.ts deleted file mode 100644 index b461aa55d..000000000 --- a/server/lib/activitypub/send/http.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { buildDigest, signJsonLDObject } from '@server/helpers/peertube-crypto' -import { ACTIVITY_PUB, HTTP_SIGNATURE } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { getServerActor } from '@server/models/application/application' -import { MActor } from '@server/types/models' -import { ContextType } from '@shared/models/activitypub/context' -import { activityPubContextify } from '../context' - -type Payload = { body: T, contextType: ContextType, signatureActorId?: number } - -async function computeBody ( - payload: Payload -): Promise { - let body = payload.body - - if (payload.signatureActorId) { - const actorSignature = await ActorModel.load(payload.signatureActorId) - if (!actorSignature) throw new Error('Unknown signature actor id.') - - body = await signAndContextify(actorSignature, payload.body, payload.contextType) - } - - return body -} - -async function buildSignedRequestOptions (options: { - signatureActorId?: number - hasPayload: boolean -}) { - let actor: MActor | null - - if (options.signatureActorId) { - actor = await ActorModel.load(options.signatureActorId) - if (!actor) throw new Error('Unknown signature actor id.') - } else { - // We need to sign the request, so use the server - actor = await getServerActor() - } - - const keyId = actor.url - return { - algorithm: HTTP_SIGNATURE.ALGORITHM, - authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, - keyId, - key: actor.privateKey, - headers: options.hasPayload - ? HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD - : HTTP_SIGNATURE.HEADERS_TO_SIGN_WITHOUT_PAYLOAD - } -} - -function buildGlobalHeaders (body: any) { - return { - 'digest': buildDigest(body), - 'content-type': 'application/activity+json', - 'accept': ACTIVITY_PUB.ACCEPT_HEADER - } -} - -async function signAndContextify (byActor: MActor, data: T, contextType: ContextType | null) { - const activity = contextType - ? await activityPubContextify(data, contextType) - : data - - return signJsonLDObject(byActor, activity) -} - -export { - buildGlobalHeaders, - computeBody, - buildSignedRequestOptions, - signAndContextify -} diff --git a/server/lib/activitypub/send/index.ts b/server/lib/activitypub/send/index.ts deleted file mode 100644 index 852ea2e74..000000000 --- a/server/lib/activitypub/send/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export * from './http' -export * from './send-accept' -export * from './send-announce' -export * from './send-create' -export * from './send-delete' -export * from './send-follow' -export * from './send-like' -export * from './send-reject' -export * from './send-undo' -export * from './send-update' diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts deleted file mode 100644 index 4c9bcbb0b..000000000 --- a/server/lib/activitypub/send/send-accept.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { ActivityAccept, ActivityFollow } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorFollowActors } from '../../../types/models' -import { getLocalActorFollowAcceptActivityPubUrl } from '../url' -import { buildFollowActivity } from './send-follow' -import { unicastTo } from './shared/send-utils' - -function sendAccept (actorFollow: MActorFollowActors) { - const follower = actorFollow.ActorFollower - const me = actorFollow.ActorFollowing - - if (!follower.serverId) { // This should never happen - logger.warn('Do not sending accept to local follower.') - return - } - - logger.info('Creating job to accept follower %s.', follower.url) - - const followData = buildFollowActivity(actorFollow.url, follower, me) - - const url = getLocalActorFollowAcceptActivityPubUrl(actorFollow) - const data = buildAcceptActivity(url, me, followData) - - return unicastTo({ - data, - byActor: me, - toActorUrl: follower.inboxUrl, - contextType: 'Accept' - }) -} - -// --------------------------------------------------------------------------- - -export { - sendAccept -} - -// --------------------------------------------------------------------------- - -function buildAcceptActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityAccept { - return { - type: 'Accept', - id: url, - actor: byActor.url, - object: followActivityData - } -} diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts deleted file mode 100644 index 6c078b047..000000000 --- a/server/lib/activitypub/send/send-announce.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAnnounce, ActivityAudience } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActorLight, MVideo } from '../../../types/models' -import { MVideoShare } from '../../../types/models/video' -import { audiencify, getAudience } from '../audience' -import { getActorsInvolvedInVideo, getAudienceFromFollowersOf } from './shared' -import { broadcastToFollowers } from './shared/send-utils' - -async function buildAnnounceWithVideoAudience ( - byActor: MActorLight, - videoShare: MVideoShare, - video: MVideo, - t: Transaction -) { - const announcedObject = video.url - - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) - - const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) - - return { activity, actorsInvolvedInVideo } -} - -async function sendVideoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { - const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) - - logger.info('Creating job to send announce %s.', videoShare.url) - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolvedInVideo, - transaction, - actorsException: [ byActor ], - contextType: 'Announce' - }) -} - -function buildAnnounceActivity (url: string, byActor: MActorLight, object: string, audience?: ActivityAudience): ActivityAnnounce { - if (!audience) audience = getAudience(byActor) - - return audiencify({ - type: 'Announce' as 'Announce', - id: url, - actor: byActor.url, - object - }, audience) -} - -// --------------------------------------------------------------------------- - -export { - sendVideoAnnounce, - buildAnnounceActivity, - buildAnnounceWithVideoAudience -} diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts deleted file mode 100644 index 2cd4db14d..000000000 --- a/server/lib/activitypub/send/send-create.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { - ActivityAudience, - ActivityCreate, - ActivityCreateObject, - ContextType, - VideoCommentObject, - VideoPlaylistPrivacy, - VideoPrivacy -} from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { - MActorLight, - MCommentOwnerVideo, - MLocalVideoViewerWithWatchSections, - MVideoAccountLight, - MVideoAP, - MVideoPlaylistFull, - MVideoRedundancyFileVideo, - MVideoRedundancyStreamingPlaylistVideo -} from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { - broadcastToActors, - broadcastToFollowers, - getActorsInvolvedInVideo, - getAudienceFromFollowersOf, - getVideoCommentAudience, - sendVideoActivityToOrigin, - sendVideoRelatedActivity, - unicastTo -} from './shared' - -const lTags = loggerTagsFactory('ap', 'create') - -async function sendCreateVideo (video: MVideoAP, transaction: Transaction) { - if (!video.hasPrivacyForFederation()) return undefined - - logger.info('Creating job to send video creation of %s.', video.url, lTags(video.uuid)) - - const byActor = video.VideoChannel.Account.Actor - const videoObject = await video.toActivityPubObject() - - const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) - const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience) - - return broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf: [ byActor ], - transaction, - contextType: 'Video' - }) -} - -async function sendCreateCacheFile ( - byActor: MActorLight, - video: MVideoAccountLight, - fileRedundancy: MVideoRedundancyStreamingPlaylistVideo | MVideoRedundancyFileVideo -) { - logger.info('Creating job to send file cache of %s.', fileRedundancy.url, lTags(video.uuid)) - - return sendVideoRelatedCreateActivity({ - byActor, - video, - url: fileRedundancy.url, - object: fileRedundancy.toActivityPubObject(), - contextType: 'CacheFile' - }) -} - -async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) { - logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid)) - - const byActor = await getServerActor() - - const activityBuilder = (audience: ActivityAudience) => { - return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' }) -} - -async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) { - if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined - - logger.info('Creating job to send create video playlist of %s.', playlist.url, lTags(playlist.uuid)) - - const byActor = playlist.OwnerAccount.Actor - const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) - - const object = await playlist.toActivityPubObject(null, transaction) - const createActivity = buildCreateActivity(playlist.url, byActor, object, audience) - - const serverActor = await getServerActor() - const toFollowersOf = [ byActor, serverActor ] - - if (playlist.VideoChannel) toFollowersOf.push(playlist.VideoChannel.Actor) - - return broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf, - transaction, - contextType: 'Playlist' - }) -} - -async function sendCreateVideoComment (comment: MCommentOwnerVideo, transaction: Transaction) { - logger.info('Creating job to send comment %s.', comment.url) - - const isOrigin = comment.Video.isOwned() - - const byActor = comment.Account.Actor - const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, transaction) - const commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject - - const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, transaction) - // Add the actor that commented too - actorsInvolvedInComment.push(byActor) - - const parentsCommentActors = threadParentComments.filter(c => !c.isDeleted()) - .map(c => c.Account.Actor) - - let audience: ActivityAudience - if (isOrigin) { - audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) - } else { - audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors)) - } - - const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) - - // This was a reply, send it to the parent actors - const actorsException = [ byActor ] - await broadcastToActors({ - data: createActivity, - byActor, - toActors: parentsCommentActors, - transaction, - actorsException, - contextType: 'Comment' - }) - - // Broadcast to our followers - await broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf: [ byActor ], - transaction, - contextType: 'Comment' - }) - - // Send to actors involved in the comment - if (isOrigin) { - return broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf: actorsInvolvedInComment, - transaction, - actorsException, - contextType: 'Comment' - }) - } - - // Send to origin - return transaction.afterCommit(() => { - return unicastTo({ - data: createActivity, - byActor, - toActorUrl: comment.Video.VideoChannel.Account.Actor.getSharedInbox(), - contextType: 'Comment' - }) - }) -} - -function buildCreateActivity ( - url: string, - byActor: MActorLight, - object: T, - audience?: ActivityAudience -): ActivityCreate { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - type: 'Create' as 'Create', - id: url + '/activity', - actor: byActor.url, - object: typeof object === 'string' - ? object - : audiencify(object, audience) - }, - audience - ) -} - -// --------------------------------------------------------------------------- - -export { - sendCreateVideo, - buildCreateActivity, - sendCreateVideoComment, - sendCreateVideoPlaylist, - sendCreateCacheFile, - sendCreateWatchAction -} - -// --------------------------------------------------------------------------- - -async function sendVideoRelatedCreateActivity (options: { - byActor: MActorLight - video: MVideoAccountLight - url: string - object: any - contextType: ContextType - transaction?: Transaction -}) { - const activityBuilder = (audience: ActivityAudience) => { - return buildCreateActivity(options.url, options.byActor, options.object, audience) - } - - return sendVideoRelatedActivity(activityBuilder, options) -} diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts deleted file mode 100644 index 0d85d9001..000000000 --- a/server/lib/activitypub/send/send-delete.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { ActivityAudience, ActivityDelete } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { ActorModel } from '../../../models/actor/actor' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoShareModel } from '../../../models/video/video-share' -import { MActorUrl } from '../../../types/models' -import { MCommentOwnerVideo, MVideoAccountLight, MVideoPlaylistFullSummary } from '../../../types/models/video' -import { audiencify } from '../audience' -import { getDeleteActivityPubUrl } from '../url' -import { getActorsInvolvedInVideo, getVideoCommentAudience } from './shared' -import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './shared/send-utils' - -async function sendDeleteVideo (video: MVideoAccountLight, transaction: Transaction) { - logger.info('Creating job to broadcast delete of video %s.', video.url) - - const byActor = video.VideoChannel.Account.Actor - - const activityBuilder = (audience: ActivityAudience) => { - const url = getDeleteActivityPubUrl(video.url) - - return buildDeleteActivity(url, video.url, byActor, audience) - } - - return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'Delete', transaction }) -} - -async function sendDeleteActor (byActor: ActorModel, transaction: Transaction) { - logger.info('Creating job to broadcast delete of actor %s.', byActor.url) - - const url = getDeleteActivityPubUrl(byActor.url) - const activity = buildDeleteActivity(url, byActor.url, byActor) - - const actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) - - // In case the actor did not have any videos - const serverActor = await getServerActor() - actorsInvolved.push(serverActor) - - actorsInvolved.push(byActor) - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolved, - contextType: 'Delete', - transaction - }) -} - -async function sendDeleteVideoComment (videoComment: MCommentOwnerVideo, transaction: Transaction) { - logger.info('Creating job to send delete of comment %s.', videoComment.url) - - const isVideoOrigin = videoComment.Video.isOwned() - - const url = getDeleteActivityPubUrl(videoComment.url) - const byActor = videoComment.isOwned() - ? videoComment.Account.Actor - : videoComment.Video.VideoChannel.Account.Actor - - const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, transaction) - const threadParentCommentsFiltered = threadParentComments.filter(c => !c.isDeleted()) - - const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, transaction) - actorsInvolvedInComment.push(byActor) // Add the actor that commented the video - - const audience = getVideoCommentAudience(videoComment, threadParentCommentsFiltered, actorsInvolvedInComment, isVideoOrigin) - const activity = buildDeleteActivity(url, videoComment.url, byActor, audience) - - // This was a reply, send it to the parent actors - const actorsException = [ byActor ] - await broadcastToActors({ - data: activity, - byActor, - toActors: threadParentCommentsFiltered.map(c => c.Account.Actor), - transaction, - contextType: 'Delete', - actorsException - }) - - // Broadcast to our followers - await broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: [ byActor ], - contextType: 'Delete', - transaction - }) - - // Send to actors involved in the comment - if (isVideoOrigin) { - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolvedInComment, - transaction, - contextType: 'Delete', - actorsException - }) - } - - // Send to origin - return transaction.afterCommit(() => { - return unicastTo({ - data: activity, - byActor, - toActorUrl: videoComment.Video.VideoChannel.Account.Actor.getSharedInbox(), - contextType: 'Delete' - }) - }) -} - -async function sendDeleteVideoPlaylist (videoPlaylist: MVideoPlaylistFullSummary, transaction: Transaction) { - logger.info('Creating job to send delete of playlist %s.', videoPlaylist.url) - - const byActor = videoPlaylist.OwnerAccount.Actor - - const url = getDeleteActivityPubUrl(videoPlaylist.url) - const activity = buildDeleteActivity(url, videoPlaylist.url, byActor) - - const serverActor = await getServerActor() - const toFollowersOf = [ byActor, serverActor ] - - if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf, - contextType: 'Delete', - transaction - }) -} - -// --------------------------------------------------------------------------- - -export { - sendDeleteVideo, - sendDeleteActor, - sendDeleteVideoComment, - sendDeleteVideoPlaylist -} - -// --------------------------------------------------------------------------- - -function buildDeleteActivity (url: string, object: string, byActor: MActorUrl, audience?: ActivityAudience): ActivityDelete { - const activity = { - type: 'Delete' as 'Delete', - id: url, - actor: byActor.url, - object - } - - if (audience) return audiencify(activity, audience) - - return activity -} diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts deleted file mode 100644 index 959e74823..000000000 --- a/server/lib/activitypub/send/send-dislike.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityDislike } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getVideoDislikeActivityPubUrlByLocalActor } from '../url' -import { sendVideoActivityToOrigin } from './shared/send-utils' - -function sendDislike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { - logger.info('Creating job to dislike %s.', video.url) - - const activityBuilder = (audience: ActivityAudience) => { - const url = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) - - return buildDislikeActivity(url, byActor, video, audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) -} - -function buildDislikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityDislike { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - id: url, - type: 'Dislike' as 'Dislike', - actor: byActor.url, - object: video.url - }, - audience - ) -} - -// --------------------------------------------------------------------------- - -export { - sendDislike, - buildDislikeActivity -} diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts deleted file mode 100644 index 138eb5adc..000000000 --- a/server/lib/activitypub/send/send-flag.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityFlag } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MAbuseAP, MAccountLight, MActor } from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getLocalAbuseActivityPubUrl } from '../url' -import { unicastTo } from './shared/send-utils' - -function sendAbuse (byActor: MActor, abuse: MAbuseAP, flaggedAccount: MAccountLight, t: Transaction) { - if (!flaggedAccount.Actor.serverId) return // Local user - - const url = getLocalAbuseActivityPubUrl(abuse) - - logger.info('Creating job to send abuse %s.', url) - - // Custom audience, we only send the abuse to the origin instance - const audience = { to: [ flaggedAccount.Actor.url ], cc: [] } - const flagActivity = buildFlagActivity(url, byActor, abuse, audience) - - return t.afterCommit(() => { - return unicastTo({ - data: flagActivity, - byActor, - toActorUrl: flaggedAccount.Actor.getSharedInbox(), - contextType: 'Flag' - }) - }) -} - -function buildFlagActivity (url: string, byActor: MActor, abuse: MAbuseAP, audience: ActivityAudience): ActivityFlag { - if (!audience) audience = getAudience(byActor) - - const activity = { id: url, actor: byActor.url, ...abuse.toActivityPubObject() } - - return audiencify(activity, audience) -} - -// --------------------------------------------------------------------------- - -export { - sendAbuse -} diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts deleted file mode 100644 index 57501dadb..000000000 --- a/server/lib/activitypub/send/send-follow.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityFollow } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorFollowActors } from '../../../types/models' -import { unicastTo } from './shared/send-utils' - -function sendFollow (actorFollow: MActorFollowActors, t: Transaction) { - const me = actorFollow.ActorFollower - const following = actorFollow.ActorFollowing - - // Same server as ours - if (!following.serverId) return - - logger.info('Creating job to send follow request to %s.', following.url) - - const data = buildFollowActivity(actorFollow.url, me, following) - - return t.afterCommit(() => { - return unicastTo({ data, byActor: me, toActorUrl: following.inboxUrl, contextType: 'Follow' }) - }) -} - -function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow { - return { - type: 'Follow', - id: url, - actor: byActor.url, - object: targetActor.url - } -} - -// --------------------------------------------------------------------------- - -export { - sendFollow, - buildFollowActivity -} diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts deleted file mode 100644 index 46c9fdec9..000000000 --- a/server/lib/activitypub/send/send-like.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityLike } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getVideoLikeActivityPubUrlByLocalActor } from '../url' -import { sendVideoActivityToOrigin } from './shared/send-utils' - -function sendLike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { - logger.info('Creating job to like %s.', video.url) - - const activityBuilder = (audience: ActivityAudience) => { - const url = getVideoLikeActivityPubUrlByLocalActor(byActor, video) - - return buildLikeActivity(url, byActor, video, audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) -} - -function buildLikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityLike { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - id: url, - type: 'Like' as 'Like', - actor: byActor.url, - object: video.url - }, - audience - ) -} - -// --------------------------------------------------------------------------- - -export { - sendLike, - buildLikeActivity -} diff --git a/server/lib/activitypub/send/send-reject.ts b/server/lib/activitypub/send/send-reject.ts deleted file mode 100644 index a5f8c2ecf..000000000 --- a/server/lib/activitypub/send/send-reject.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { ActivityFollow, ActivityReject } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor } from '../../../types/models' -import { getLocalActorFollowRejectActivityPubUrl } from '../url' -import { buildFollowActivity } from './send-follow' -import { unicastTo } from './shared/send-utils' - -function sendReject (followUrl: string, follower: MActor, following: MActor) { - if (!follower.serverId) { // This should never happen - logger.warn('Do not sending reject to local follower.') - return - } - - logger.info('Creating job to reject follower %s.', follower.url) - - const followData = buildFollowActivity(followUrl, follower, following) - - const url = getLocalActorFollowRejectActivityPubUrl() - const data = buildRejectActivity(url, following, followData) - - return unicastTo({ data, byActor: following, toActorUrl: follower.inboxUrl, contextType: 'Reject' }) -} - -// --------------------------------------------------------------------------- - -export { - sendReject -} - -// --------------------------------------------------------------------------- - -function buildRejectActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityReject { - return { - type: 'Reject', - id: url, - actor: byActor.url, - object: followActivityData - } -} diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts deleted file mode 100644 index b0b48c9c4..000000000 --- a/server/lib/activitypub/send/send-undo.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityDislike, ActivityLike, ActivityUndo, ActivityUndoObject, ContextType } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' -import { - MActor, - MActorAudience, - MActorFollowActors, - MActorLight, - MVideo, - MVideoAccountLight, - MVideoRedundancyVideo, - MVideoShare -} from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getUndoActivityPubUrl, getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from '../url' -import { buildAnnounceWithVideoAudience } from './send-announce' -import { buildCreateActivity } from './send-create' -import { buildDislikeActivity } from './send-dislike' -import { buildFollowActivity } from './send-follow' -import { buildLikeActivity } from './send-like' -import { broadcastToFollowers, sendVideoActivityToOrigin, sendVideoRelatedActivity, unicastTo } from './shared/send-utils' - -function sendUndoFollow (actorFollow: MActorFollowActors, t: Transaction) { - const me = actorFollow.ActorFollower - const following = actorFollow.ActorFollowing - - // Same server as ours - if (!following.serverId) return - - logger.info('Creating job to send an unfollow request to %s.', following.url) - - const undoUrl = getUndoActivityPubUrl(actorFollow.url) - - const followActivity = buildFollowActivity(actorFollow.url, me, following) - const undoActivity = undoActivityData(undoUrl, me, followActivity) - - t.afterCommit(() => { - return unicastTo({ - data: undoActivity, - byActor: me, - toActorUrl: following.inboxUrl, - contextType: 'Follow' - }) - }) -} - -// --------------------------------------------------------------------------- - -async function sendUndoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { - logger.info('Creating job to undo announce %s.', videoShare.url) - - const undoUrl = getUndoActivityPubUrl(videoShare.url) - - const { activity: announce, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) - const undoActivity = undoActivityData(undoUrl, byActor, announce) - - return broadcastToFollowers({ - data: undoActivity, - byActor, - toFollowersOf: actorsInvolvedInVideo, - transaction, - actorsException: [ byActor ], - contextType: 'Announce' - }) -} - -async function sendUndoCacheFile (byActor: MActor, redundancyModel: MVideoRedundancyVideo, transaction: Transaction) { - logger.info('Creating job to undo cache file %s.', redundancyModel.url) - - const associatedVideo = redundancyModel.getVideo() - if (!associatedVideo) { - logger.warn('Cannot send undo activity for redundancy %s: no video files associated.', redundancyModel.url) - return - } - - const video = await VideoModel.loadFull(associatedVideo.id) - const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) - - return sendUndoVideoRelatedActivity({ - byActor, - video, - url: redundancyModel.url, - activity: createActivity, - contextType: 'CacheFile', - transaction - }) -} - -// --------------------------------------------------------------------------- - -async function sendUndoLike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { - logger.info('Creating job to undo a like of video %s.', video.url) - - const likeUrl = getVideoLikeActivityPubUrlByLocalActor(byActor, video) - const likeActivity = buildLikeActivity(likeUrl, byActor, video) - - return sendUndoVideoRateToOriginActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t }) -} - -async function sendUndoDislike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { - logger.info('Creating job to undo a dislike of video %s.', video.url) - - const dislikeUrl = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) - const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) - - return sendUndoVideoRateToOriginActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) -} - -// --------------------------------------------------------------------------- - -export { - sendUndoFollow, - sendUndoLike, - sendUndoDislike, - sendUndoAnnounce, - sendUndoCacheFile -} - -// --------------------------------------------------------------------------- - -function undoActivityData ( - url: string, - byActor: MActorAudience, - object: T, - audience?: ActivityAudience -): ActivityUndo { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - type: 'Undo' as 'Undo', - id: url, - actor: byActor.url, - object - }, - audience - ) -} - -async function sendUndoVideoRelatedActivity (options: { - byActor: MActor - video: MVideoAccountLight - url: string - activity: ActivityUndoObject - contextType: ContextType - transaction: Transaction -}) { - const activityBuilder = (audience: ActivityAudience) => { - const undoUrl = getUndoActivityPubUrl(options.url) - - return undoActivityData(undoUrl, options.byActor, options.activity, audience) - } - - return sendVideoRelatedActivity(activityBuilder, options) -} - -async function sendUndoVideoRateToOriginActivity (options: { - byActor: MActor - video: MVideoAccountLight - url: string - activity: ActivityLike | ActivityDislike - transaction: Transaction -}) { - const activityBuilder = (audience: ActivityAudience) => { - const undoUrl = getUndoActivityPubUrl(options.url) - - return undoActivityData(undoUrl, options.byActor, options.activity, audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { ...options, contextType: 'Rate' }) -} diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts deleted file mode 100644 index f3fb741c6..000000000 --- a/server/lib/activitypub/send/send-update.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { ActivityAudience, ActivityUpdate, ActivityUpdateObject, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { AccountModel } from '../../../models/account/account' -import { VideoModel } from '../../../models/video/video' -import { VideoShareModel } from '../../../models/video/video-share' -import { - MAccountDefault, - MActor, - MActorLight, - MChannelDefault, - MVideoAPLight, - MVideoPlaylistFull, - MVideoRedundancyVideo -} from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getUpdateActivityPubUrl } from '../url' -import { getActorsInvolvedInVideo } from './shared' -import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils' - -async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) { - if (!videoArg.hasPrivacyForFederation()) return undefined - - const video = await videoArg.lightAPToFullAP(transaction) - - logger.info('Creating job to update video %s.', video.url) - - const byActor = overriddenByActor || video.VideoChannel.Account.Actor - - const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) - - const videoObject = await video.toActivityPubObject() - const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) - - const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience) - - const actorsInvolved = await getActorsInvolvedInVideo(video, transaction) - if (overriddenByActor) actorsInvolved.push(overriddenByActor) - - return broadcastToFollowers({ - data: updateActivity, - byActor, - toFollowersOf: actorsInvolved, - contextType: 'Video', - transaction - }) -} - -async function sendUpdateActor (accountOrChannel: MChannelDefault | MAccountDefault, transaction: Transaction) { - const byActor = accountOrChannel.Actor - - logger.info('Creating job to update actor %s.', byActor.url) - - const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) - const accountOrChannelObject = await (accountOrChannel as any).toActivityPubObject() // FIXME: typescript bug? - const audience = getAudience(byActor) - const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience) - - let actorsInvolved: MActor[] - if (accountOrChannel instanceof AccountModel) { - // Actors that shared my videos are involved too - actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) - } else { - // Actors that shared videos of my channel are involved too - actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, transaction) - } - - actorsInvolved.push(byActor) - - return broadcastToFollowers({ - data: updateActivity, - byActor, - toFollowersOf: actorsInvolved, - transaction, - contextType: 'Actor' - }) -} - -async function sendUpdateCacheFile (byActor: MActorLight, redundancyModel: MVideoRedundancyVideo) { - logger.info('Creating job to update cache file %s.', redundancyModel.url) - - const associatedVideo = redundancyModel.getVideo() - if (!associatedVideo) { - logger.warn('Cannot send update activity for redundancy %s: no video files associated.', redundancyModel.url) - return - } - - const video = await VideoModel.loadFull(associatedVideo.id) - - const activityBuilder = (audience: ActivityAudience) => { - const redundancyObject = redundancyModel.toActivityPubObject() - const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) - - return buildUpdateActivity(url, byActor, redundancyObject, audience) - } - - return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'CacheFile' }) -} - -async function sendUpdateVideoPlaylist (videoPlaylist: MVideoPlaylistFull, transaction: Transaction) { - if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined - - const byActor = videoPlaylist.OwnerAccount.Actor - - logger.info('Creating job to update video playlist %s.', videoPlaylist.url) - - const url = getUpdateActivityPubUrl(videoPlaylist.url, videoPlaylist.updatedAt.toISOString()) - - const object = await videoPlaylist.toActivityPubObject(null, transaction) - const audience = getAudience(byActor, videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC) - - const updateActivity = buildUpdateActivity(url, byActor, object, audience) - - const serverActor = await getServerActor() - const toFollowersOf = [ byActor, serverActor ] - - if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) - - return broadcastToFollowers({ - data: updateActivity, - byActor, - toFollowersOf, - transaction, - contextType: 'Playlist' - }) -} - -// --------------------------------------------------------------------------- - -export { - sendUpdateActor, - sendUpdateVideo, - sendUpdateCacheFile, - sendUpdateVideoPlaylist -} - -// --------------------------------------------------------------------------- - -function buildUpdateActivity ( - url: string, - byActor: MActorLight, - object: ActivityUpdateObject, - audience?: ActivityAudience -): ActivityUpdate { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - type: 'Update' as 'Update', - id: url, - actor: byActor.url, - object: audiencify(object, audience) - }, - audience - ) -} diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts deleted file mode 100644 index bf3451603..000000000 --- a/server/lib/activitypub/send/send-view.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { Transaction } from 'sequelize' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models' -import { ActivityAudience, ActivityView } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { audiencify, getAudience } from '../audience' -import { getLocalVideoViewActivityPubUrl } from '../url' -import { sendVideoRelatedActivity } from './shared/send-utils' - -type ViewType = 'view' | 'viewer' - -async function sendView (options: { - byActor: MActorLight - type: ViewType - video: MVideoImmutable - viewerIdentifier: string - transaction?: Transaction -}) { - const { byActor, type, video, viewerIdentifier, transaction } = options - - logger.info('Creating job to send %s of %s.', type, video.url) - - const activityBuilder = (audience: ActivityAudience) => { - const url = getLocalVideoViewActivityPubUrl(byActor, video, viewerIdentifier) - - return buildViewActivity({ url, byActor, video, audience, type }) - } - - return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View', parallelizable: true }) -} - -// --------------------------------------------------------------------------- - -export { - sendView -} - -// --------------------------------------------------------------------------- - -function buildViewActivity (options: { - url: string - byActor: MActorAudience - video: MVideoUrl - type: ViewType - audience?: ActivityAudience -}): ActivityView { - const { url, byActor, type, video, audience = getAudience(byActor) } = options - - return audiencify( - { - id: url, - type: 'View' as 'View', - actor: byActor.url, - object: video.url, - - expires: type === 'viewer' - ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString() - : undefined - }, - audience - ) -} diff --git a/server/lib/activitypub/send/shared/audience-utils.ts b/server/lib/activitypub/send/shared/audience-utils.ts deleted file mode 100644 index 2f6b0741d..000000000 --- a/server/lib/activitypub/send/shared/audience-utils.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Transaction } from 'sequelize' -import { ACTIVITY_PUB } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { VideoModel } from '@server/models/video/video' -import { VideoShareModel } from '@server/models/video/video-share' -import { MActorFollowersUrl, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' -import { ActivityAudience } from '@shared/models' - -function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { - return { - to: [ accountActor.url ], - cc: actorsInvolvedInVideo.map(a => a.followersUrl) - } -} - -function getVideoCommentAudience ( - videoComment: MCommentOwnerVideo, - threadParentComments: MCommentOwner[], - actorsInvolvedInVideo: MActorFollowersUrl[], - isOrigin = false -): ActivityAudience { - const to = [ ACTIVITY_PUB.PUBLIC ] - const cc: string[] = [] - - // Owner of the video we comment - if (isOrigin === false) { - cc.push(videoComment.Video.VideoChannel.Account.Actor.url) - } - - // Followers of the poster - cc.push(videoComment.Account.Actor.followersUrl) - - // Send to actors we reply to - for (const parentComment of threadParentComments) { - if (parentComment.isDeleted()) continue - - cc.push(parentComment.Account.Actor.url) - } - - return { - to, - cc: cc.concat(actorsInvolvedInVideo.map(a => a.followersUrl)) - } -} - -function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[]): ActivityAudience { - return { - to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), - cc: [] - } -} - -async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { - const actors = await VideoShareModel.listActorIdsAndFollowerUrlsByShare(video.id, t) - - const videoAll = video as VideoModel - - const videoActor = videoAll.VideoChannel?.Account - ? videoAll.VideoChannel.Account.Actor - : await ActorModel.loadAccountActorFollowerUrlByVideoId(video.id, t) - - actors.push(videoActor) - - return actors -} - -// --------------------------------------------------------------------------- - -export { - getOriginVideoAudience, - getActorsInvolvedInVideo, - getAudienceFromFollowersOf, - getVideoCommentAudience -} diff --git a/server/lib/activitypub/send/shared/index.ts b/server/lib/activitypub/send/shared/index.ts deleted file mode 100644 index bda579115..000000000 --- a/server/lib/activitypub/send/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './audience-utils' -export * from './send-utils' diff --git a/server/lib/activitypub/send/shared/send-utils.ts b/server/lib/activitypub/send/shared/send-utils.ts deleted file mode 100644 index 2bc1ef8f5..000000000 --- a/server/lib/activitypub/send/shared/send-utils.ts +++ /dev/null @@ -1,291 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache' -import { getServerActor } from '@server/models/application/application' -import { Activity, ActivityAudience, ActivitypubHttpBroadcastPayload } from '@shared/models' -import { ContextType } from '@shared/models/activitypub/context' -import { afterCommitIfTransaction } from '../../../../helpers/database-utils' -import { logger } from '../../../../helpers/logger' -import { ActorModel } from '../../../../models/actor/actor' -import { ActorFollowModel } from '../../../../models/actor/actor-follow' -import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../../types/models' -import { JobQueue } from '../../../job-queue' -import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getOriginVideoAudience } from './audience-utils' - -async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { - byActor: MActorLight - video: MVideoImmutable | MVideoAccountLight - contextType: ContextType - parallelizable?: boolean - transaction?: Transaction -}) { - const { byActor, video, transaction, contextType, parallelizable } = options - - // Send to origin - if (video.isOwned() === false) { - return sendVideoActivityToOrigin(activityBuilder, options) - } - - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) - - // Send to followers - const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) - const activity = activityBuilder(audience) - - const actorsException = [ byActor ] - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolvedInVideo, - transaction, - actorsException, - parallelizable, - contextType - }) -} - -async function sendVideoActivityToOrigin (activityBuilder: (audience: ActivityAudience) => Activity, options: { - byActor: MActorLight - video: MVideoImmutable | MVideoAccountLight - contextType: ContextType - - actorsInvolvedInVideo?: MActorLight[] - transaction?: Transaction -}) { - const { byActor, video, actorsInvolvedInVideo, transaction, contextType } = options - - if (video.isOwned()) throw new Error('Cannot send activity to owned video origin ' + video.url) - - let accountActor: MActorLight = (video as MVideoAccountLight).VideoChannel?.Account?.Actor - if (!accountActor) accountActor = await ActorModel.loadAccountActorByVideoId(video.id, transaction) - - const audience = getOriginVideoAudience(accountActor, actorsInvolvedInVideo) - const activity = activityBuilder(audience) - - return afterCommitIfTransaction(transaction, () => { - return unicastTo({ - data: activity, - byActor, - toActorUrl: accountActor.getSharedInbox(), - contextType - }) - }) -} - -// --------------------------------------------------------------------------- - -async function forwardVideoRelatedActivity ( - activity: Activity, - t: Transaction, - followersException: MActorWithInboxes[], - video: MVideoId -) { - // Mastodon does not add our announces in audience, so we forward to them manually - const additionalActors = await getActorsInvolvedInVideo(video, t) - const additionalFollowerUrls = additionalActors.map(a => a.followersUrl) - - return forwardActivity(activity, t, followersException, additionalFollowerUrls) -} - -async function forwardActivity ( - activity: Activity, - t: Transaction, - followersException: MActorWithInboxes[] = [], - additionalFollowerUrls: string[] = [] -) { - logger.info('Forwarding activity %s.', activity.id) - - const to = activity.to || [] - const cc = activity.cc || [] - - const followersUrls = additionalFollowerUrls - for (const dest of to.concat(cc)) { - if (dest.endsWith('/followers')) { - followersUrls.push(dest) - } - } - - const toActorFollowers = await ActorModel.listByFollowersUrls(followersUrls, t) - const uris = await computeFollowerUris(toActorFollowers, followersException, t) - - if (uris.length === 0) { - logger.info('0 followers for %s, no forwarding.', toActorFollowers.map(a => a.id).join(', ')) - return undefined - } - - logger.debug('Creating forwarding job.', { uris }) - - const payload: ActivitypubHttpBroadcastPayload = { - uris, - body: activity, - contextType: null - } - return afterCommitIfTransaction(t, () => JobQueue.Instance.createJobAsync({ type: 'activitypub-http-broadcast', payload })) -} - -// --------------------------------------------------------------------------- - -async function broadcastToFollowers (options: { - data: any - byActor: MActorId - toFollowersOf: MActorId[] - transaction: Transaction - contextType: ContextType - - parallelizable?: boolean - actorsException?: MActorWithInboxes[] -}) { - const { data, byActor, toFollowersOf, transaction, contextType, actorsException = [], parallelizable } = options - - const uris = await computeFollowerUris(toFollowersOf, actorsException, transaction) - - return afterCommitIfTransaction(transaction, () => { - return broadcastTo({ - uris, - data, - byActor, - parallelizable, - contextType - }) - }) -} - -async function broadcastToActors (options: { - data: any - byActor: MActorId - toActors: MActor[] - transaction: Transaction - contextType: ContextType - actorsException?: MActorWithInboxes[] -}) { - const { data, byActor, toActors, transaction, contextType, actorsException = [] } = options - - const uris = await computeUris(toActors, actorsException) - - return afterCommitIfTransaction(transaction, () => { - return broadcastTo({ - uris, - data, - byActor, - contextType - }) - }) -} - -function broadcastTo (options: { - uris: string[] - data: any - byActor: MActorId - contextType: ContextType - parallelizable?: boolean // default to false -}) { - const { uris, data, byActor, contextType, parallelizable } = options - - if (uris.length === 0) return undefined - - const broadcastUris: string[] = [] - const unicastUris: string[] = [] - - // Bad URIs could be slow to respond, prefer to process them in a dedicated queue - for (const uri of uris) { - if (ActorFollowHealthCache.Instance.isBadInbox(uri)) { - unicastUris.push(uri) - } else { - broadcastUris.push(uri) - } - } - - logger.debug('Creating broadcast job.', { broadcastUris, unicastUris }) - - if (broadcastUris.length !== 0) { - const payload = { - uris: broadcastUris, - signatureActorId: byActor.id, - body: data, - contextType - } - - JobQueue.Instance.createJobAsync({ - type: parallelizable - ? 'activitypub-http-broadcast-parallel' - : 'activitypub-http-broadcast', - - payload - }) - } - - for (const unicastUri of unicastUris) { - const payload = { - uri: unicastUri, - signatureActorId: byActor.id, - body: data, - contextType - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) - } -} - -function unicastTo (options: { - data: any - byActor: MActorId - toActorUrl: string - contextType: ContextType -}) { - const { data, byActor, toActorUrl, contextType } = options - - logger.debug('Creating unicast job.', { uri: toActorUrl }) - - const payload = { - uri: toActorUrl, - signatureActorId: byActor.id, - body: data, - contextType - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) -} - -// --------------------------------------------------------------------------- - -export { - broadcastToFollowers, - unicastTo, - forwardActivity, - broadcastToActors, - sendVideoActivityToOrigin, - forwardVideoRelatedActivity, - sendVideoRelatedActivity -} - -// --------------------------------------------------------------------------- - -async function computeFollowerUris (toFollowersOf: MActorId[], actorsException: MActorWithInboxes[], t: Transaction) { - const toActorFollowerIds = toFollowersOf.map(a => a.id) - - const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) - const sharedInboxesException = await buildSharedInboxesException(actorsException) - - return result.data.filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) -} - -async function computeUris (toActors: MActor[], actorsException: MActorWithInboxes[] = []) { - const serverActor = await getServerActor() - const targetUrls = toActors - .filter(a => a.id !== serverActor.id) // Don't send to ourselves - .map(a => a.getSharedInbox()) - - const toActorSharedInboxesSet = new Set(targetUrls) - - const sharedInboxesException = await buildSharedInboxesException(actorsException) - return Array.from(toActorSharedInboxesSet) - .filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) -} - -async function buildSharedInboxesException (actorsException: MActorWithInboxes[]) { - const serverActor = await getServerActor() - - return actorsException - .map(f => f.getSharedInbox()) - .concat([ serverActor.sharedInboxUrl ]) -} diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts deleted file mode 100644 index 792a73f2a..000000000 --- a/server/lib/activitypub/share.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { map } from 'bluebird' -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { logger, loggerTagsFactory } from '../../helpers/logger' -import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' -import { VideoShareModel } from '../../models/video/video-share' -import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' -import { fetchAP, getAPId } from './activity' -import { getOrCreateAPActor } from './actors' -import { sendUndoAnnounce, sendVideoAnnounce } from './send' -import { checkUrlsSameHost, getLocalVideoAnnounceActivityPubUrl } from './url' - -const lTags = loggerTagsFactory('share') - -async function shareVideoByServerAndChannel (video: MVideoAccountLight, t: Transaction) { - if (!video.hasPrivacyForFederation()) return undefined - - return Promise.all([ - shareByServer(video, t), - shareByVideoChannel(video, t) - ]) -} - -async function changeVideoChannelShare ( - video: MVideoAccountLight, - oldVideoChannel: MChannelActorLight, - t: Transaction -) { - logger.info( - 'Updating video channel of video %s: %s -> %s.', video.uuid, oldVideoChannel.name, video.VideoChannel.name, - lTags(video.uuid) - ) - - await undoShareByVideoChannel(video, oldVideoChannel, t) - - await shareByVideoChannel(video, t) -} - -async function addVideoShares (shareUrls: string[], video: MVideoId) { - await map(shareUrls, async shareUrl => { - try { - await addVideoShare(shareUrl, video) - } catch (err) { - logger.warn('Cannot add share %s.', shareUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -export { - changeVideoChannelShare, - addVideoShares, - shareVideoByServerAndChannel -} - -// --------------------------------------------------------------------------- - -async function addVideoShare (shareUrl: string, video: MVideoId) { - const { body } = await fetchAP(shareUrl) - if (!body?.actor) throw new Error('Body or body actor is invalid') - - const actorUrl = getAPId(body.actor) - if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { - throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) - } - - const actor = await getOrCreateAPActor(actorUrl) - - const entry = { - actorId: actor.id, - videoId: video.id, - url: shareUrl - } - - await VideoShareModel.upsert(entry) -} - -async function shareByServer (video: MVideo, t: Transaction) { - const serverActor = await getServerActor() - - const serverShareUrl = getLocalVideoAnnounceActivityPubUrl(serverActor, video) - const [ serverShare ] = await VideoShareModel.findOrCreate({ - defaults: { - actorId: serverActor.id, - videoId: video.id, - url: serverShareUrl - }, - where: { - url: serverShareUrl - }, - transaction: t - }) - - return sendVideoAnnounce(serverActor, serverShare, video, t) -} - -async function shareByVideoChannel (video: MVideoAccountLight, t: Transaction) { - const videoChannelShareUrl = getLocalVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) - const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ - defaults: { - actorId: video.VideoChannel.actorId, - videoId: video.id, - url: videoChannelShareUrl - }, - where: { - url: videoChannelShareUrl - }, - transaction: t - }) - - return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) -} - -async function undoShareByVideoChannel (video: MVideo, oldVideoChannel: MChannelActorLight, t: Transaction) { - // Load old share - const oldShare = await VideoShareModel.load(oldVideoChannel.actorId, video.id, t) - if (!oldShare) return new Error('Cannot find old video channel share ' + oldVideoChannel.actorId + ' for video ' + video.id) - - await sendUndoAnnounce(oldVideoChannel.Actor, oldShare, video, t) - await oldShare.destroy({ transaction: t }) -} diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts deleted file mode 100644 index 5cdac71bf..000000000 --- a/server/lib/activitypub/url.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants' -import { - MAbuseFull, - MAbuseId, - MActor, - MActorFollow, - MActorId, - MActorUrl, - MCommentId, - MLocalVideoViewer, - MVideoId, - MVideoPlaylistElement, - MVideoUrl, - MVideoUUID, - MVideoWithHost -} from '../../types/models' -import { MVideoFileVideoUUID } from '../../types/models/video/video-file' -import { MVideoPlaylist, MVideoPlaylistUUID } from '../../types/models/video/video-playlist' -import { MStreamingPlaylist } from '../../types/models/video/video-streaming-playlist' - -function getLocalVideoActivityPubUrl (video: MVideoUUID) { - return WEBSERVER.URL + '/videos/watch/' + video.uuid -} - -function getLocalVideoPlaylistActivityPubUrl (videoPlaylist: MVideoPlaylist) { - return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid -} - -function getLocalVideoPlaylistElementActivityPubUrl (videoPlaylist: MVideoPlaylistUUID, videoPlaylistElement: MVideoPlaylistElement) { - return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid + '/videos/' + videoPlaylistElement.id -} - -function getLocalVideoCacheFileActivityPubUrl (videoFile: MVideoFileVideoUUID) { - const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' - - return `${WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` -} - -function getLocalVideoCacheStreamingPlaylistActivityPubUrl (video: MVideoUUID, playlist: MStreamingPlaylist) { - return `${WEBSERVER.URL}/redundancy/streaming-playlists/${playlist.getStringType()}/${video.uuid}` -} - -function getLocalVideoCommentActivityPubUrl (video: MVideoUUID, videoComment: MCommentId) { - return WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id -} - -function getLocalVideoChannelActivityPubUrl (videoChannelName: string) { - return WEBSERVER.URL + '/video-channels/' + videoChannelName -} - -function getLocalAccountActivityPubUrl (accountName: string) { - return WEBSERVER.URL + '/accounts/' + accountName -} - -function getLocalAbuseActivityPubUrl (abuse: MAbuseId) { - return WEBSERVER.URL + '/admin/abuses/' + abuse.id -} - -function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId, viewerIdentifier: string) { - return byActor.url + '/views/videos/' + video.id + '/' + viewerIdentifier -} - -function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { - return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid -} - -function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { - return byActor.url + '/likes/' + video.id -} - -function getVideoDislikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { - return byActor.url + '/dislikes/' + video.id -} - -function getLocalVideoSharesActivityPubUrl (video: MVideoUrl) { - return video.url + '/announces' -} - -function getLocalVideoCommentsActivityPubUrl (video: MVideoUrl) { - return video.url + '/comments' -} - -function getLocalVideoLikesActivityPubUrl (video: MVideoUrl) { - return video.url + '/likes' -} - -function getLocalVideoDislikesActivityPubUrl (video: MVideoUrl) { - return video.url + '/dislikes' -} - -function getLocalActorFollowActivityPubUrl (follower: MActor, following: MActorId) { - return follower.url + '/follows/' + following.id -} - -function getLocalActorFollowAcceptActivityPubUrl (actorFollow: MActorFollow) { - return WEBSERVER.URL + '/accepts/follows/' + actorFollow.id -} - -function getLocalActorFollowRejectActivityPubUrl () { - return WEBSERVER.URL + '/rejects/follows/' + new Date().toISOString() -} - -function getLocalVideoAnnounceActivityPubUrl (byActor: MActorId, video: MVideoUrl) { - return video.url + '/announces/' + byActor.id -} - -function getDeleteActivityPubUrl (originalUrl: string) { - return originalUrl + '/delete' -} - -function getUpdateActivityPubUrl (originalUrl: string, updatedAt: string) { - return originalUrl + '/updates/' + updatedAt -} - -function getUndoActivityPubUrl (originalUrl: string) { - return originalUrl + '/undo' -} - -// --------------------------------------------------------------------------- - -function getAbuseTargetUrl (abuse: MAbuseFull) { - return abuse.VideoAbuse?.Video?.url || - abuse.VideoCommentAbuse?.VideoComment?.url || - abuse.FlaggedAccount.Actor.url -} - -// --------------------------------------------------------------------------- - -function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string, scheme?: string) { - if (!scheme) scheme = REMOTE_SCHEME.HTTP - - const host = video.VideoChannel.Actor.Server.host - - return scheme + '://' + host + path -} - -// --------------------------------------------------------------------------- - -function checkUrlsSameHost (url1: string, url2: string) { - const idHost = new URL(url1).host - const actorHost = new URL(url2).host - - return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() -} - -// --------------------------------------------------------------------------- - -export { - getLocalVideoActivityPubUrl, - getLocalVideoPlaylistActivityPubUrl, - getLocalVideoPlaylistElementActivityPubUrl, - getLocalVideoCacheFileActivityPubUrl, - getLocalVideoCacheStreamingPlaylistActivityPubUrl, - getLocalVideoCommentActivityPubUrl, - getLocalVideoChannelActivityPubUrl, - getLocalAccountActivityPubUrl, - getLocalAbuseActivityPubUrl, - getLocalActorFollowActivityPubUrl, - getLocalActorFollowAcceptActivityPubUrl, - getLocalVideoAnnounceActivityPubUrl, - getUpdateActivityPubUrl, - getUndoActivityPubUrl, - getVideoLikeActivityPubUrlByLocalActor, - getLocalVideoViewActivityPubUrl, - getVideoDislikeActivityPubUrlByLocalActor, - getLocalActorFollowRejectActivityPubUrl, - getDeleteActivityPubUrl, - getLocalVideoSharesActivityPubUrl, - getLocalVideoCommentsActivityPubUrl, - getLocalVideoLikesActivityPubUrl, - getLocalVideoDislikesActivityPubUrl, - getLocalVideoViewerActivityPubUrl, - - getAbuseTargetUrl, - checkUrlsSameHost, - buildRemoteVideoBaseUrl -} diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts deleted file mode 100644 index b861be5bd..000000000 --- a/server/lib/activitypub/video-comments.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { map } from 'bluebird' - -import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' -import { logger } from '../../helpers/logger' -import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' -import { VideoCommentModel } from '../../models/video/video-comment' -import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' -import { isRemoteVideoCommentAccepted } from '../moderation' -import { Hooks } from '../plugins/hooks' -import { fetchAP } from './activity' -import { getOrCreateAPActor } from './actors' -import { checkUrlsSameHost } from './url' -import { getOrCreateAPVideo } from './videos' - -type ResolveThreadParams = { - url: string - comments?: MCommentOwner[] - isVideo?: boolean - commentCreated?: boolean -} -type ResolveThreadResult = Promise<{ video: MVideoAccountLightBlacklistAllFiles, comment: MCommentOwnerVideo, commentCreated: boolean }> - -async function addVideoComments (commentUrls: string[]) { - return map(commentUrls, async commentUrl => { - try { - await resolveThread({ url: commentUrl, isVideo: false }) - } catch (err) { - logger.warn('Cannot resolve thread %s.', commentUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { - const { url, isVideo } = params - - if (params.commentCreated === undefined) params.commentCreated = false - if (params.comments === undefined) params.comments = [] - - // If it is not a video, or if we don't know if it's a video, try to get the thread from DB - if (isVideo === false || isVideo === undefined) { - const result = await resolveCommentFromDB(params) - if (result) return result - } - - try { - // If it is a video, or if we don't know if it's a video - if (isVideo === true || isVideo === undefined) { - // Keep await so we catch the exception - return await tryToResolveThreadFromVideo(params) - } - } catch (err) { - logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) - } - - return resolveRemoteParentComment(params) -} - -export { - addVideoComments, - resolveThread -} - -// --------------------------------------------------------------------------- - -async function resolveCommentFromDB (params: ResolveThreadParams) { - const { url, comments, commentCreated } = params - - const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) - if (!commentFromDatabase) return undefined - - let parentComments = comments.concat([ commentFromDatabase ]) - - // Speed up things and resolve directly the thread - if (commentFromDatabase.InReplyToVideoComment) { - const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') - - parentComments = parentComments.concat(data) - } - - return resolveThread({ - url: commentFromDatabase.Video.url, - comments: parentComments, - isVideo: true, - commentCreated - }) -} - -async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { - const { url, comments, commentCreated } = params - - // Maybe it's a reply to a video? - // If yes, it's done: we resolved all the thread - const syncParam = { rates: true, shares: true, comments: false, refreshVideo: false } - const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam }) - - if (video.isOwned() && !video.hasPrivacyForFederation()) { - throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') - } - - let resultComment: MCommentOwnerVideo - if (comments.length !== 0) { - const firstReply = comments[comments.length - 1] as MCommentOwnerVideo - firstReply.inReplyToCommentId = null - firstReply.originCommentId = null - firstReply.videoId = video.id - firstReply.changed('updatedAt', true) - firstReply.Video = video - - if (await isRemoteCommentAccepted(firstReply) !== true) { - return undefined - } - - comments[comments.length - 1] = await firstReply.save() - - for (let i = comments.length - 2; i >= 0; i--) { - const comment = comments[i] as MCommentOwnerVideo - comment.originCommentId = firstReply.id - comment.inReplyToCommentId = comments[i + 1].id - comment.videoId = video.id - comment.changed('updatedAt', true) - comment.Video = video - - if (await isRemoteCommentAccepted(comment) !== true) { - return undefined - } - - comments[i] = await comment.save() - } - - resultComment = comments[0] as MCommentOwnerVideo - } - - return { video, comment: resultComment, commentCreated } -} - -async function resolveRemoteParentComment (params: ResolveThreadParams) { - const { url, comments } = params - - if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { - throw new Error('Recursion limit reached when resolving a thread') - } - - const { body } = await fetchAP(url) - - if (sanitizeAndCheckVideoCommentObject(body) === false) { - throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) - } - - const actorUrl = body.attributedTo - if (!actorUrl && body.type !== 'Tombstone') throw new Error('Miss attributed to in comment') - - if (actorUrl && checkUrlsSameHost(url, actorUrl) !== true) { - throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`) - } - - if (checkUrlsSameHost(body.id, url) !== true) { - throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`) - } - - const actor = actorUrl - ? await getOrCreateAPActor(actorUrl, 'all') - : null - - const comment = new VideoCommentModel({ - url: body.id, - text: body.content ? body.content : '', - videoId: null, - accountId: actor ? actor.Account.id : null, - inReplyToCommentId: null, - originCommentId: null, - createdAt: new Date(body.published), - updatedAt: new Date(body.updated), - deletedAt: body.deleted ? new Date(body.deleted) : null - }) as MCommentOwner - comment.Account = actor ? actor.Account : null - - return resolveThread({ - url: body.inReplyTo, - comments: comments.concat([ comment ]), - commentCreated: true - }) -} - -async function isRemoteCommentAccepted (comment: MComment) { - // Already created - if (comment.id) return true - - const acceptParameters = { - comment - } - - const acceptedResult = await Hooks.wrapFun( - isRemoteVideoCommentAccepted, - acceptParameters, - 'filter:activity-pub.remote-video-comment.create.accept.result' - ) - - if (!acceptedResult || acceptedResult.accepted !== true) { - logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters }) - - return false - } - - return true -} diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts deleted file mode 100644 index 2e7920f4e..000000000 --- a/server/lib/activitypub/video-rates.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Transaction } from 'sequelize' -import { VideoRateType } from '../../../shared/models/videos' -import { MAccountActor, MActorUrl, MVideoAccountLight, MVideoFullLight, MVideoId } from '../../types/models' -import { sendLike, sendUndoDislike, sendUndoLike } from './send' -import { sendDislike } from './send/send-dislike' -import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' -import { federateVideoIfNeeded } from './videos' - -async function sendVideoRateChange ( - account: MAccountActor, - video: MVideoFullLight, - likes: number, - dislikes: number, - t: Transaction -) { - if (video.isOwned()) return federateVideoIfNeeded(video, false, t) - - return sendVideoRateChangeToOrigin(account, video, likes, dislikes, t) -} - -function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVideoId) { - return rateType === 'like' - ? getVideoLikeActivityPubUrlByLocalActor(actor, video) - : getVideoDislikeActivityPubUrlByLocalActor(actor, video) -} - -// --------------------------------------------------------------------------- - -export { - getLocalRateUrl, - sendVideoRateChange -} - -// --------------------------------------------------------------------------- - -async function sendVideoRateChangeToOrigin ( - account: MAccountActor, - video: MVideoAccountLight, - likes: number, - dislikes: number, - t: Transaction -) { - // Local video, we don't need to send like - if (video.isOwned()) return - - const actor = account.Actor - - // Keep the order: first we undo and then we create - - // Undo Like - if (likes < 0) await sendUndoLike(actor, video, t) - // Undo Dislike - if (dislikes < 0) await sendUndoDislike(actor, video, t) - - // Like - if (likes > 0) await sendLike(actor, video, t) - // Dislike - if (dislikes > 0) await sendDislike(actor, video, t) -} diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts deleted file mode 100644 index d7e251153..000000000 --- a/server/lib/activitypub/videos/federate.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { MVideoAP, MVideoAPLight } from '@server/types/models' -import { sendCreateVideo, sendUpdateVideo } from '../send' -import { shareVideoByServerAndChannel } from '../share' - -async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) { - const video = videoArg as MVideoAP - - if ( - // Check this is not a blacklisted video, or unfederated blacklisted video - (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) && - // Check the video is public/unlisted and published - video.hasPrivacyForFederation() && video.hasStateForFederation() - ) { - const video = await videoArg.lightAPToFullAP(transaction) - - if (isNewVideo) { - // Now we'll add the video's meta data to our followers - await sendCreateVideo(video, transaction) - await shareVideoByServerAndChannel(video, transaction) - } else { - await sendUpdateVideo(video, transaction) - } - } -} - -export { - federateVideoIfNeeded -} diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts deleted file mode 100644 index 288c506ee..000000000 --- a/server/lib/activitypub/videos/get.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { JobQueue } from '@server/lib/job-queue' -import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders' -import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' -import { APObjectId } from '@shared/models' -import { getAPId } from '../activity' -import { refreshVideoIfNeeded } from './refresh' -import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' - -type GetVideoResult = Promise<{ - video: T - created: boolean - autoBlacklisted?: boolean -}> - -type GetVideoParamAll = { - videoObject: APObjectId - syncParam?: SyncParam - fetchType?: 'all' - allowRefresh?: boolean -} - -type GetVideoParamImmutable = { - videoObject: APObjectId - syncParam?: SyncParam - fetchType: 'only-immutable-attributes' - allowRefresh: false -} - -type GetVideoParamOther = { - videoObject: APObjectId - syncParam?: SyncParam - fetchType?: 'all' | 'only-video' - allowRefresh?: boolean -} - -function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult -function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult -function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult - -async function getOrCreateAPVideo ( - options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther -): GetVideoResult { - // Default params - const syncParam = options.syncParam || { rates: true, shares: true, comments: true, refreshVideo: false } - const fetchType = options.fetchType || 'all' - const allowRefresh = options.allowRefresh !== false - - // Get video url - const videoUrl = getAPId(options.videoObject) - let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) - - if (videoFromDatabase) { - if (allowRefresh === true) { - // Typings ensure allowRefresh === false in only-immutable-attributes fetch type - videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) - } - - return { video: videoFromDatabase, created: false } - } - - const { videoObject } = await fetchRemoteVideo(videoUrl) - if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) - - // videoUrl is just an alias/rediraction, so process object id instead - if (videoObject.id !== videoUrl) return getOrCreateAPVideo({ ...options, fetchType: 'all', videoObject }) - - try { - const creator = new APVideoCreator(videoObject) - const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator)) - - await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) - - return { video: videoCreated, created: true, autoBlacklisted } - } catch (err) { - // Maybe a concurrent getOrCreateAPVideo call created this video - if (err.name === 'SequelizeUniqueConstraintError') { - const alreadyCreatedVideo = await loadVideoByUrl(videoUrl, fetchType) - if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } - - logger.error('Cannot create video %s because of SequelizeUniqueConstraintError error, but cannot find it in database.', videoUrl) - } - - throw err - } -} - -// --------------------------------------------------------------------------- - -export { - getOrCreateAPVideo -} - -// --------------------------------------------------------------------------- - -async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) { - if (!video.isOutdated()) return video - - const refreshOptions = { - video, - fetchedType: fetchType, - syncParam - } - - if (syncParam.refreshVideo === true) { - return refreshVideoIfNeeded(refreshOptions) - } - - await JobQueue.Instance.createJob({ - type: 'activitypub-refresher', - payload: { type: 'video', url: video.url } - }) - - return video -} diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts deleted file mode 100644 index b22062598..000000000 --- a/server/lib/activitypub/videos/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './federate' -export * from './get' -export * from './refresh' -export * from './updater' diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts deleted file mode 100644 index 9f952a218..000000000 --- a/server/lib/activitypub/videos/refresh.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { VideoLoadByUrlType } from '@server/lib/model-loaders' -import { VideoModel } from '@server/models/video/video' -import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { ActorFollowHealthCache } from '../../actor-follow-health-cache' -import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' -import { APVideoUpdater } from './updater' - -async function refreshVideoIfNeeded (options: { - video: MVideoThumbnail - fetchedType: VideoLoadByUrlType - syncParam: SyncParam -}): Promise { - if (!options.video.isOutdated()) return options.video - - // We need more attributes if the argument video was fetched with not enough joints - const video = options.fetchedType === 'all' - ? options.video as MVideoAccountLightBlacklistAllFiles - : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) - - const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url) - - logger.info('Refreshing video %s.', video.url, lTags()) - - try { - const { videoObject } = await fetchRemoteVideo(video.url) - - if (videoObject === undefined) { - logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags()) - - await video.setAsRefreshed() - return video - } - - const videoUpdater = new APVideoUpdater(videoObject, video) - await videoUpdater.update() - - await syncVideoExternalAttributes(video, videoObject, options.syncParam) - - ActorFollowHealthCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) - - return video - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags()) - - // Video does not exist anymore - await video.destroy() - return undefined - } - - logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags() }) - - ActorFollowHealthCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) - - // Don't refresh in loop - await video.setAsRefreshed() - return video - } -} - -// --------------------------------------------------------------------------- - -export { - refreshVideoIfNeeded -} diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts deleted file mode 100644 index 98c2f58eb..000000000 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ /dev/null @@ -1,190 +0,0 @@ -import { CreationAttributes, Transaction } from 'sequelize/types' -import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' -import { logger, LoggerTagsFn } from '@server/helpers/logger' -import { updateRemoteVideoThumbnail } from '@server/lib/thumbnail' -import { setVideoTags } from '@server/lib/video' -import { StoryboardModel } from '@server/models/video/storyboard' -import { VideoCaptionModel } from '@server/models/video/video-caption' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { - MStreamingPlaylistFiles, - MStreamingPlaylistFilesVideo, - MVideoCaption, - MVideoFile, - MVideoFullLight, - MVideoThumbnail -} from '@server/types/models' -import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' -import { findOwner, getOrCreateAPActor } from '../../actors' -import { - getCaptionAttributesFromObject, - getFileAttributesFromUrl, - getLiveAttributesFromObject, - getPreviewFromIcons, - getStoryboardAttributeFromObject, - getStreamingPlaylistAttributesFromObject, - getTagsFromObject, - getThumbnailFromIcons -} from './object-to-model-attributes' -import { getTrackerUrls, setVideoTrackers } from './trackers' - -export abstract class APVideoAbstractBuilder { - protected abstract videoObject: VideoObject - protected abstract lTags: LoggerTagsFn - - protected async getOrCreateVideoChannelFromVideoObject () { - const channel = await findOwner(this.videoObject.id, this.videoObject.attributedTo, 'Group') - if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url) - - return getOrCreateAPActor(channel.id, 'all') - } - - protected async setThumbnail (video: MVideoThumbnail, t?: Transaction) { - const miniatureIcon = getThumbnailFromIcons(this.videoObject) - if (!miniatureIcon) { - logger.warn('Cannot find thumbnail in video object', { object: this.videoObject }) - return undefined - } - - const miniatureModel = updateRemoteVideoThumbnail({ - fileUrl: miniatureIcon.url, - video, - type: ThumbnailType.MINIATURE, - size: miniatureIcon, - onDisk: false // Lazy download remote thumbnails - }) - - await video.addAndSaveThumbnail(miniatureModel, t) - } - - protected async setPreview (video: MVideoFullLight, t?: Transaction) { - const previewIcon = getPreviewFromIcons(this.videoObject) - if (!previewIcon) return - - const previewModel = updateRemoteVideoThumbnail({ - fileUrl: previewIcon.url, - video, - type: ThumbnailType.PREVIEW, - size: previewIcon, - onDisk: false // Lazy download remote previews - }) - - await video.addAndSaveThumbnail(previewModel, t) - } - - protected async setTags (video: MVideoFullLight, t: Transaction) { - const tags = getTagsFromObject(this.videoObject) - await setVideoTags({ video, tags, transaction: t }) - } - - protected async setTrackers (video: MVideoFullLight, t: Transaction) { - const trackers = getTrackerUrls(this.videoObject, video) - await setVideoTrackers({ video, trackers, transaction: t }) - } - - protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) { - const existingCaptions = await VideoCaptionModel.listVideoCaptions(video.id, t) - - let captionsToCreate = getCaptionAttributesFromObject(video, this.videoObject) - .map(a => new VideoCaptionModel(a) as MVideoCaption) - - for (const existingCaption of existingCaptions) { - // Only keep captions that do not already exist - const filtered = captionsToCreate.filter(c => !c.isEqual(existingCaption)) - - // This caption already exists, we don't need to destroy and create it - if (filtered.length !== captionsToCreate.length) { - captionsToCreate = filtered - continue - } - - // Destroy this caption that does not exist anymore - await existingCaption.destroy({ transaction: t }) - } - - for (const captionToCreate of captionsToCreate) { - await captionToCreate.save({ transaction: t }) - } - } - - protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) { - const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t) - if (existingStoryboard) await existingStoryboard.destroy({ transaction: t }) - - const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject) - if (!storyboardAttributes) return - - return StoryboardModel.create(storyboardAttributes, { transaction: t }) - } - - protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { - const attributes = getLiveAttributesFromObject(video, this.videoObject) - const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) - - video.VideoLive = videoLive - } - - protected async setWebVideoFiles (video: MVideoFullLight, t: Transaction) { - const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url) - const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) - - // Remove video files that do not exist anymore - await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t) - - // Update or add other one - const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) - video.VideoFiles = await Promise.all(upsertTasks) - } - - protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) { - const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject) - const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) - - // Remove video playlists that do not exist anymore - await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) - - const oldPlaylists = video.VideoStreamingPlaylists - video.VideoStreamingPlaylists = [] - - for (const playlistAttributes of streamingPlaylistAttributes) { - const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) - streamingPlaylistModel.Video = video - - await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t) - - video.VideoStreamingPlaylists.push(streamingPlaylistModel) - } - } - - private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes, t: Transaction) { - const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) - - return streamingPlaylist as MStreamingPlaylistFilesVideo - } - - private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType) { - const playlist = oldPlaylists.find(s => s.type === type) - if (!playlist) return [] - - return playlist.VideoFiles - } - - private async setStreamingPlaylistFiles ( - oldPlaylists: MStreamingPlaylistFiles[], - playlistModel: MStreamingPlaylistFilesVideo, - tagObjects: ActivityTagObject[], - t: Transaction - ) { - const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type) - - const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) - - await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t) - - // Update or add other one - const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) - playlistModel.VideoFiles = await Promise.all(upsertTasks) - } -} diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts deleted file mode 100644 index e44fd0d52..000000000 --- a/server/lib/activitypub/videos/shared/creator.ts +++ /dev/null @@ -1,65 +0,0 @@ - -import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' -import { Hooks } from '@server/lib/plugins/hooks' -import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' -import { VideoModel } from '@server/models/video/video' -import { MVideoFullLight, MVideoThumbnail } from '@server/types/models' -import { VideoObject } from '@shared/models' -import { APVideoAbstractBuilder } from './abstract-builder' -import { getVideoAttributesFromObject } from './object-to-model-attributes' - -export class APVideoCreator extends APVideoAbstractBuilder { - protected lTags: LoggerTagsFn - - constructor (protected readonly videoObject: VideoObject) { - super() - - this.lTags = loggerTagsFactory('ap', 'video', 'create', this.videoObject.uuid, this.videoObject.id) - } - - async create () { - logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags()) - - const channelActor = await this.getOrCreateVideoChannelFromVideoObject() - const channel = channelActor.VideoChannel - - const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) - const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail - - const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { - const videoCreated = await video.save({ transaction: t }) as MVideoFullLight - videoCreated.VideoChannel = channel - - await this.setThumbnail(videoCreated, t) - await this.setPreview(videoCreated, t) - await this.setWebVideoFiles(videoCreated, t) - await this.setStreamingPlaylists(videoCreated, t) - await this.setTags(videoCreated, t) - await this.setTrackers(videoCreated, t) - await this.insertOrReplaceCaptions(videoCreated, t) - await this.insertOrReplaceLive(videoCreated, t) - await this.insertOrReplaceStoryboard(videoCreated, t) - - // We added a video in this channel, set it as updated - await channel.setAsUpdated(t) - - const autoBlacklisted = await autoBlacklistVideoIfNeeded({ - video: videoCreated, - user: undefined, - isRemote: true, - isNew: true, - isNewFile: true, - transaction: t - }) - - logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) - - Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) - - return { autoBlacklisted, videoCreated } - }) - - return { autoBlacklisted, videoCreated } - } -} diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts deleted file mode 100644 index 951403493..000000000 --- a/server/lib/activitypub/videos/shared/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './abstract-builder' -export * from './creator' -export * from './object-to-model-attributes' -export * from './trackers' -export * from './url-to-object' -export * from './video-sync-attributes' diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts deleted file mode 100644 index 6cbe72e27..000000000 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { maxBy, minBy } from 'lodash' -import { decode as magnetUriDecode } from 'magnet-uri' -import { basename, extname } from 'path' -import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' -import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' -import { logger } from '@server/helpers/logger' -import { getExtFromMimetype } from '@server/helpers/video' -import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' -import { generateTorrentFileName } from '@server/lib/paths' -import { VideoCaptionModel } from '@server/models/video/video-caption' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { FilteredModelAttributes } from '@server/types' -import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoId } from '@server/types/models' -import { - ActivityHashTagObject, - ActivityMagnetUrlObject, - ActivityPlaylistSegmentHashesObject, - ActivityPlaylistUrlObject, - ActivityTagObject, - ActivityUrlObject, - ActivityVideoUrlObject, - VideoObject, - VideoPrivacy, - VideoStreamingPlaylistType -} from '@shared/models' -import { getDurationFromActivityStream } from '../../activity' -import { isArray } from '@server/helpers/custom-validators/misc' -import { generateImageFilename } from '@server/helpers/image-utils' -import { arrayify } from '@shared/core-utils' - -function getThumbnailFromIcons (videoObject: VideoObject) { - let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) - // Fallback if there are not valid icons - if (validIcons.length === 0) validIcons = videoObject.icon - - return minBy(validIcons, 'width') -} - -function getPreviewFromIcons (videoObject: VideoObject) { - const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth) - - return maxBy(validIcons, 'width') -} - -function getTagsFromObject (videoObject: VideoObject) { - return videoObject.tag - .filter(isAPHashTagObject) - .map(t => t.name) -} - -function getFileAttributesFromUrl ( - videoOrPlaylist: MVideo | MStreamingPlaylistVideo, - urls: (ActivityTagObject | ActivityUrlObject)[] -) { - const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] - - if (fileUrls.length === 0) return [] - - const attributes: FilteredModelAttributes[] = [] - for (const fileUrl of fileUrls) { - // Fetch associated magnet uri - const magnet = urls.filter(isAPMagnetUrlObject) - .find(u => u.height === fileUrl.height) - - if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) - - const parsed = magnetUriDecode(magnet.href) - if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { - throw new Error('Cannot parse magnet URI ' + magnet.href) - } - - const torrentUrl = Array.isArray(parsed.xs) - ? parsed.xs[0] - : parsed.xs - - // Fetch associated metadata url, if any - const metadata = urls.filter(isAPVideoFileUrlMetadataObject) - .find(u => { - return u.height === fileUrl.height && - u.fps === fileUrl.fps && - u.rel.includes(fileUrl.mediaType) - }) - - const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) - const resolution = fileUrl.height - const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id - const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null - - const attribute = { - extname, - infoHash: parsed.infoHash, - resolution, - size: fileUrl.size, - fps: fileUrl.fps || -1, - metadataUrl: metadata?.href, - - // Use the name of the remote file because we don't proxify video file requests - filename: basename(fileUrl.href), - fileUrl: fileUrl.href, - - torrentUrl, - // Use our own torrent name since we proxify torrent requests - torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), - - // This is a video file owned by a video or by a streaming playlist - videoId, - videoStreamingPlaylistId - } - - attributes.push(attribute) - } - - return attributes -} - -function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject) { - const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] - if (playlistUrls.length === 0) return [] - - const attributes: (FilteredModelAttributes & { tagAPObject?: ActivityTagObject[] })[] = [] - for (const playlistUrlObject of playlistUrls) { - const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject) - - const files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] - - if (!segmentsSha256UrlObject) { - logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) - continue - } - - const attribute = { - type: VideoStreamingPlaylistType.HLS, - - playlistFilename: basename(playlistUrlObject.href), - playlistUrl: playlistUrlObject.href, - - segmentsSha256Filename: basename(segmentsSha256UrlObject.href), - segmentsSha256Url: segmentsSha256UrlObject.href, - - p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), - p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, - videoId: video.id, - - tagAPObject: playlistUrlObject.tag - } - - attributes.push(attribute) - } - - return attributes -} - -function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) { - return { - saveReplay: videoObject.liveSaveReplay, - permanentLive: videoObject.permanentLive, - latencyMode: videoObject.latencyMode, - videoId: video.id - } -} - -function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) { - return videoObject.subtitleLanguage.map(c => ({ - videoId: video.id, - filename: VideoCaptionModel.generateCaptionName(c.identifier), - language: c.identifier, - fileUrl: c.url - })) -} - -function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) { - if (!isArray(videoObject.preview)) return undefined - - const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard')) - if (!storyboard) return undefined - - const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg') - - return { - filename: generateImageFilename(extname(url.href)), - totalHeight: url.height, - totalWidth: url.width, - spriteHeight: url.tileHeight, - spriteWidth: url.tileWidth, - spriteDuration: getDurationFromActivityStream(url.tileDuration), - fileUrl: url.href, - videoId: video.id - } -} - -function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { - const privacy = to.includes(ACTIVITY_PUB.PUBLIC) - ? VideoPrivacy.PUBLIC - : VideoPrivacy.UNLISTED - - const language = videoObject.language?.identifier - - const category = videoObject.category - ? parseInt(videoObject.category.identifier, 10) - : undefined - - const licence = videoObject.licence - ? parseInt(videoObject.licence.identifier, 10) - : undefined - - const description = videoObject.content || null - const support = videoObject.support || null - - return { - name: videoObject.name, - uuid: videoObject.uuid, - url: videoObject.id, - category, - licence, - language, - description, - support, - nsfw: videoObject.sensitive, - commentsEnabled: videoObject.commentsEnabled, - downloadEnabled: videoObject.downloadEnabled, - waitTranscoding: videoObject.waitTranscoding, - isLive: videoObject.isLiveBroadcast, - state: videoObject.state, - channelId: videoChannel.id, - duration: getDurationFromActivityStream(videoObject.duration), - createdAt: new Date(videoObject.published), - publishedAt: new Date(videoObject.published), - - originallyPublishedAt: videoObject.originallyPublishedAt - ? new Date(videoObject.originallyPublishedAt) - : null, - - inputFileUpdatedAt: videoObject.uploadDate - ? new Date(videoObject.uploadDate) - : null, - - updatedAt: new Date(videoObject.updated), - views: videoObject.views, - remote: true, - privacy - } -} - -// --------------------------------------------------------------------------- - -export { - getThumbnailFromIcons, - getPreviewFromIcons, - - getTagsFromObject, - - getFileAttributesFromUrl, - getStreamingPlaylistAttributesFromObject, - - getLiveAttributesFromObject, - getCaptionAttributesFromObject, - getStoryboardAttributeFromObject, - - getVideoAttributesFromObject -} - -// --------------------------------------------------------------------------- - -function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { - const urlMediaType = url.mediaType - - return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/') -} - -function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject { - return url && url.mediaType === 'application/x-mpegURL' -} - -function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { - return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json' -} - -function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { - return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' -} - -function isAPHashTagObject (url: any): url is ActivityHashTagObject { - return url && url.type === 'Hashtag' -} diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts deleted file mode 100644 index 2418f45c2..000000000 --- a/server/lib/activitypub/videos/shared/trackers.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos' -import { isArray } from '@server/helpers/custom-validators/misc' -import { REMOTE_SCHEME } from '@server/initializers/constants' -import { TrackerModel } from '@server/models/server/tracker' -import { MVideo, MVideoWithHost } from '@server/types/models' -import { ActivityTrackerUrlObject, VideoObject } from '@shared/models' -import { buildRemoteVideoBaseUrl } from '../../url' - -function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { - let wsFound = false - - const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u)) - .map((u: ActivityTrackerUrlObject) => { - if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true - - return u.href - }) - - if (wsFound) return trackers - - return [ - buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS), - buildRemoteVideoBaseUrl(video, '/tracker/announce') - ] -} - -async function setVideoTrackers (options: { - video: MVideo - trackers: string[] - transaction: Transaction -}) { - const { video, trackers, transaction } = options - - const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction) - - await video.$set('Trackers', trackerInstances, { transaction }) -} - -export { - getTrackerUrls, - setVideoTrackers -} diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts deleted file mode 100644 index 7fe008419..000000000 --- a/server/lib/activitypub/videos/shared/url-to-object.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { VideoObject } from '@shared/models' -import { fetchAP } from '../../activity' -import { checkUrlsSameHost } from '../../url' - -const lTags = loggerTagsFactory('ap', 'video') - -async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { - logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl)) - - const { statusCode, body } = await fetchAP(videoUrl) - - if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { - logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) }) - - return { statusCode, videoObject: undefined } - } - - return { statusCode, videoObject: body } -} - -export { - fetchRemoteVideo -} diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts deleted file mode 100644 index 7fb933559..000000000 --- a/server/lib/activitypub/videos/shared/video-sync-attributes.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { runInReadCommittedTransaction } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { JobQueue } from '@server/lib/job-queue' -import { VideoModel } from '@server/models/video/video' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { VideoShareModel } from '@server/models/video/video-share' -import { MVideo } from '@server/types/models' -import { ActivitypubHttpFetcherPayload, ActivityPubOrderedCollection, VideoObject } from '@shared/models' -import { fetchAP } from '../../activity' -import { crawlCollectionPage } from '../../crawl' -import { addVideoShares } from '../../share' -import { addVideoComments } from '../../video-comments' - -const lTags = loggerTagsFactory('ap', 'video') - -type SyncParam = { - rates: boolean - shares: boolean - comments: boolean - refreshVideo?: boolean -} - -async function syncVideoExternalAttributes ( - video: MVideo, - fetchedVideo: VideoObject, - syncParam: Pick -) { - logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) - - const ratePromise = updateVideoRates(video, fetchedVideo) - if (syncParam.rates) await ratePromise - - await syncShares(video, fetchedVideo, syncParam.shares) - - await syncComments(video, fetchedVideo, syncParam.comments) -} - -async function updateVideoRates (video: MVideo, fetchedVideo: VideoObject) { - const [ likes, dislikes ] = await Promise.all([ - getRatesCount('like', video, fetchedVideo), - getRatesCount('dislike', video, fetchedVideo) - ]) - - return runInReadCommittedTransaction(async t => { - await VideoModel.updateRatesOf(video.id, 'like', likes, t) - await VideoModel.updateRatesOf(video.id, 'dislike', dislikes, t) - }) -} - -// --------------------------------------------------------------------------- - -export { - SyncParam, - syncVideoExternalAttributes, - updateVideoRates -} - -// --------------------------------------------------------------------------- - -async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVideo: VideoObject) { - const uri = type === 'like' - ? fetchedVideo.likes - : fetchedVideo.dislikes - - logger.info('Sync %s of video %s', type, video.url) - - const { body } = await fetchAP>(uri) - - if (isNaN(body.totalItems)) { - logger.error('Cannot sync %s of video %s, totalItems is not a number', type, video.url, { body }) - return - } - - return body.totalItems -} - -function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { - const uri = fetchedVideo.shares - - if (!isSync) { - return createJob({ uri, videoId: video.id, type: 'video-shares' }) - } - - const handler = items => addVideoShares(items, video) - const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) - - return crawlCollectionPage(uri, handler, cleaner) - .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) -} - -function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { - const uri = fetchedVideo.comments - - if (!isSync) { - return createJob({ uri, videoId: video.id, type: 'video-comments' }) - } - - const handler = items => addVideoComments(items) - const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) - - return crawlCollectionPage(uri, handler, cleaner) - .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) -} - -function createJob (payload: ActivitypubHttpFetcherPayload) { - return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) -} diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts deleted file mode 100644 index acb087895..000000000 --- a/server/lib/activitypub/videos/updater.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' -import { Notifier } from '@server/lib/notifier' -import { PeerTubeSocket } from '@server/lib/peertube-socket' -import { Hooks } from '@server/lib/plugins/hooks' -import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' -import { VideoLiveModel } from '@server/models/video/video-live' -import { MActor, MChannelAccountLight, MChannelId, MVideoAccountLightBlacklistAllFiles, MVideoFullLight } from '@server/types/models' -import { VideoObject, VideoPrivacy } from '@shared/models' -import { APVideoAbstractBuilder, getVideoAttributesFromObject, updateVideoRates } from './shared' - -export class APVideoUpdater extends APVideoAbstractBuilder { - private readonly wasPrivateVideo: boolean - private readonly wasUnlistedVideo: boolean - - private readonly oldVideoChannel: MChannelAccountLight - - protected lTags: LoggerTagsFn - - constructor ( - protected readonly videoObject: VideoObject, - private readonly video: MVideoAccountLightBlacklistAllFiles - ) { - super() - - this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE - this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED - - this.oldVideoChannel = this.video.VideoChannel - - this.lTags = loggerTagsFactory('ap', 'video', 'update', video.uuid, video.url) - } - - async update (overrideTo?: string[]) { - logger.debug( - 'Updating remote video "%s".', this.videoObject.uuid, - { videoObject: this.videoObject, ...this.lTags() } - ) - - const oldInputFileUpdatedAt = this.video.inputFileUpdatedAt - - try { - const channelActor = await this.getOrCreateVideoChannelFromVideoObject() - - const thumbnailModel = await this.setThumbnail(this.video) - - this.checkChannelUpdateOrThrow(channelActor) - - const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo) - - if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) - - await runInReadCommittedTransaction(async t => { - await this.setWebVideoFiles(videoUpdated, t) - await this.setStreamingPlaylists(videoUpdated, t) - }) - - await Promise.all([ - runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), - runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), - runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)), - runInReadCommittedTransaction(t => { - return Promise.all([ - this.setPreview(videoUpdated, t), - this.setThumbnail(videoUpdated, t) - ]) - }), - this.setOrDeleteLive(videoUpdated) - ]) - - await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) - - await autoBlacklistVideoIfNeeded({ - video: videoUpdated, - user: undefined, - isRemote: true, - isNew: false, - isNewFile: oldInputFileUpdatedAt !== videoUpdated.inputFileUpdatedAt, - transaction: undefined - }) - - await updateVideoRates(videoUpdated, this.videoObject) - - // Notify our users? - if (this.wasPrivateVideo || this.wasUnlistedVideo) { - Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) - } - - if (videoUpdated.isLive) { - PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated) - } - - Hooks.runAction('action:activity-pub.remote-video.updated', { video: videoUpdated, videoAPObject: this.videoObject }) - - logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags()) - - return videoUpdated - } catch (err) { - await this.catchUpdateError(err) - } - } - - // Check we can update the channel: we trust the remote server - private checkChannelUpdateOrThrow (newChannelActor: MActor) { - if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) { - throw new Error('Cannot check old channel/new channel validity because `serverId` is null') - } - - if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) { - throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`) - } - } - - private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) { - const to = overrideTo || this.videoObject.to - const videoData = getVideoAttributesFromObject(channel, this.videoObject, to) - this.video.name = videoData.name - this.video.uuid = videoData.uuid - this.video.url = videoData.url - this.video.category = videoData.category - this.video.licence = videoData.licence - this.video.language = videoData.language - this.video.description = videoData.description - this.video.support = videoData.support - this.video.nsfw = videoData.nsfw - this.video.commentsEnabled = videoData.commentsEnabled - this.video.downloadEnabled = videoData.downloadEnabled - this.video.waitTranscoding = videoData.waitTranscoding - this.video.state = videoData.state - this.video.duration = videoData.duration - this.video.createdAt = videoData.createdAt - this.video.publishedAt = videoData.publishedAt - this.video.originallyPublishedAt = videoData.originallyPublishedAt - this.video.inputFileUpdatedAt = videoData.inputFileUpdatedAt - this.video.privacy = videoData.privacy - this.video.channelId = videoData.channelId - this.video.views = videoData.views - this.video.isLive = videoData.isLive - - // Ensures we update the updatedAt attribute, even if main attributes did not change - this.video.changed('updatedAt', true) - - return this.video.save({ transaction }) as Promise - } - - private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) { - await this.insertOrReplaceCaptions(videoUpdated, t) - } - - private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) { - await this.insertOrReplaceStoryboard(videoUpdated, t) - } - - private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { - if (!this.video.isLive) return - - if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction) - - // Delete existing live if it exists - await VideoLiveModel.destroy({ - where: { - videoId: this.video.id - }, - transaction - }) - - videoUpdated.VideoLive = null - } - - private async catchUpdateError (err: Error) { - if (this.video !== undefined) { - await resetSequelizeInstance(this.video) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote video.', { err, ...this.lTags() }) - throw err - } -} diff --git a/server/lib/actor-follow-health-cache.ts b/server/lib/actor-follow-health-cache.ts deleted file mode 100644 index 34357a97a..000000000 --- a/server/lib/actor-follow-health-cache.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { ACTOR_FOLLOW_SCORE } from '../initializers/constants' -import { logger } from '../helpers/logger' - -// Cache follows scores, instead of writing them too often in database -// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores -class ActorFollowHealthCache { - - private static instance: ActorFollowHealthCache - - private pendingFollowsScore: { [ url: string ]: number } = {} - - private pendingBadServer = new Set() - private pendingGoodServer = new Set() - - private readonly badInboxes = new Set() - - private constructor () {} - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - updateActorFollowsHealth (goodInboxes: string[], badInboxes: string[]) { - this.badInboxes.clear() - - if (goodInboxes.length === 0 && badInboxes.length === 0) return - - logger.info( - 'Updating %d good actor follows and %d bad actor follows scores in cache.', - goodInboxes.length, badInboxes.length, { badInboxes } - ) - - for (const goodInbox of goodInboxes) { - if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0 - - this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS - } - - for (const badInbox of badInboxes) { - if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0 - - this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY - this.badInboxes.add(badInbox) - } - } - - isBadInbox (inboxUrl: string) { - return this.badInboxes.has(inboxUrl) - } - - addBadServerId (serverId: number) { - this.pendingBadServer.add(serverId) - } - - getBadFollowingServerIds () { - return Array.from(this.pendingBadServer) - } - - clearBadFollowingServerIds () { - this.pendingBadServer = new Set() - } - - addGoodServerId (serverId: number) { - this.pendingGoodServer.add(serverId) - } - - getGoodFollowingServerIds () { - return Array.from(this.pendingGoodServer) - } - - clearGoodFollowingServerIds () { - this.pendingGoodServer = new Set() - } - - getPendingFollowsScore () { - return this.pendingFollowsScore - } - - clearPendingFollowsScore () { - this.pendingFollowsScore = {} - } -} - -export { - ActorFollowHealthCache -} diff --git a/server/lib/actor-image.ts b/server/lib/actor-image.ts deleted file mode 100644 index e9bd148f6..000000000 --- a/server/lib/actor-image.ts +++ /dev/null @@ -1,14 +0,0 @@ -import maxBy from 'lodash/maxBy' - -function getBiggestActorImage (images: T[]) { - const image = maxBy(images, 'width') - - // If width is null, maxBy won't return a value - if (!image) return images[0] - - return image -} - -export { - getBiggestActorImage -} diff --git a/server/lib/auth/external-auth.ts b/server/lib/auth/external-auth.ts deleted file mode 100644 index bc5b74257..000000000 --- a/server/lib/auth/external-auth.ts +++ /dev/null @@ -1,231 +0,0 @@ - -import { - isUserAdminFlagsValid, - isUserDisplayNameValid, - isUserRoleValid, - isUserUsernameValid, - isUserVideoQuotaDailyValid, - isUserVideoQuotaValid -} from '@server/helpers/custom-validators/users' -import { logger } from '@server/helpers/logger' -import { generateRandomString } from '@server/helpers/utils' -import { PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME } from '@server/initializers/constants' -import { PluginManager } from '@server/lib/plugins/plugin-manager' -import { OAuthTokenModel } from '@server/models/oauth/oauth-token' -import { MUser } from '@server/types/models' -import { - RegisterServerAuthenticatedResult, - RegisterServerAuthPassOptions, - RegisterServerExternalAuthenticatedResult -} from '@server/types/plugins/register-server-auth.model' -import { UserAdminFlag, UserRole } from '@shared/models' -import { BypassLogin } from './oauth-model' - -export type ExternalUser = - Pick & - { displayName: string } - -// Token is the key, expiration date is the value -const authBypassTokens = new Map() - -async function onExternalUserAuthenticated (options: { - npmName: string - authName: string - authResult: RegisterServerExternalAuthenticatedResult -}) { - const { npmName, authName, authResult } = options - - if (!authResult.req || !authResult.res) { - logger.error('Cannot authenticate external user for auth %s of plugin %s: no req or res are provided.', authName, npmName) - return - } - - const { res } = authResult - - if (!isAuthResultValid(npmName, authName, authResult)) { - res.redirect('/login?externalAuthError=true') - return - } - - logger.info('Generating auth bypass token for %s in auth %s of plugin %s.', authResult.username, authName, npmName) - - const bypassToken = await generateRandomString(32) - - const expires = new Date() - expires.setTime(expires.getTime() + PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME) - - const user = buildUserResult(authResult) - authBypassTokens.set(bypassToken, { - expires, - user, - npmName, - authName, - userUpdater: authResult.userUpdater - }) - - // Cleanup expired tokens - const now = new Date() - for (const [ key, value ] of authBypassTokens) { - if (value.expires.getTime() < now.getTime()) { - authBypassTokens.delete(key) - } - } - - res.redirect(`/login?externalAuthToken=${bypassToken}&username=${user.username}`) -} - -async function getAuthNameFromRefreshGrant (refreshToken?: string) { - if (!refreshToken) return undefined - - const tokenModel = await OAuthTokenModel.loadByRefreshToken(refreshToken) - - return tokenModel?.authName -} - -async function getBypassFromPasswordGrant (username: string, password: string): Promise { - const plugins = PluginManager.Instance.getIdAndPassAuths() - const pluginAuths: { npmName?: string, registerAuthOptions: RegisterServerAuthPassOptions }[] = [] - - for (const plugin of plugins) { - const auths = plugin.idAndPassAuths - - for (const auth of auths) { - pluginAuths.push({ - npmName: plugin.npmName, - registerAuthOptions: auth - }) - } - } - - pluginAuths.sort((a, b) => { - const aWeight = a.registerAuthOptions.getWeight() - const bWeight = b.registerAuthOptions.getWeight() - - // DESC weight order - if (aWeight === bWeight) return 0 - if (aWeight < bWeight) return 1 - return -1 - }) - - const loginOptions = { - id: username, - password - } - - for (const pluginAuth of pluginAuths) { - const authOptions = pluginAuth.registerAuthOptions - const authName = authOptions.authName - const npmName = pluginAuth.npmName - - logger.debug( - 'Using auth method %s of plugin %s to login %s with weight %d.', - authName, npmName, loginOptions.id, authOptions.getWeight() - ) - - try { - const loginResult = await authOptions.login(loginOptions) - - if (!loginResult) continue - if (!isAuthResultValid(pluginAuth.npmName, authOptions.authName, loginResult)) continue - - logger.info( - 'Login success with auth method %s of plugin %s for %s.', - authName, npmName, loginOptions.id - ) - - return { - bypass: true, - pluginName: pluginAuth.npmName, - authName: authOptions.authName, - user: buildUserResult(loginResult), - userUpdater: loginResult.userUpdater - } - } catch (err) { - logger.error('Error in auth method %s of plugin %s', authOptions.authName, pluginAuth.npmName, { err }) - } - } - - return undefined -} - -function getBypassFromExternalAuth (username: string, externalAuthToken: string): BypassLogin { - const obj = authBypassTokens.get(externalAuthToken) - if (!obj) throw new Error('Cannot authenticate user with unknown bypass token') - - const { expires, user, authName, npmName } = obj - - const now = new Date() - if (now.getTime() > expires.getTime()) { - throw new Error('Cannot authenticate user with an expired external auth token') - } - - if (user.username !== username) { - throw new Error(`Cannot authenticate user ${user.username} with invalid username ${username}`) - } - - logger.info( - 'Auth success with external auth method %s of plugin %s for %s.', - authName, npmName, user.email - ) - - return { - bypass: true, - pluginName: npmName, - authName, - userUpdater: obj.userUpdater, - user - } -} - -function isAuthResultValid (npmName: string, authName: string, result: RegisterServerAuthenticatedResult) { - const returnError = (field: string) => { - logger.error('Auth method %s of plugin %s did not provide a valid %s.', authName, npmName, field, { [field]: result[field] }) - return false - } - - if (!isUserUsernameValid(result.username)) return returnError('username') - if (!result.email) return returnError('email') - - // Following fields are optional - if (result.role && !isUserRoleValid(result.role)) return returnError('role') - if (result.displayName && !isUserDisplayNameValid(result.displayName)) return returnError('displayName') - if (result.adminFlags && !isUserAdminFlagsValid(result.adminFlags)) return returnError('adminFlags') - if (result.videoQuota && !isUserVideoQuotaValid(result.videoQuota + '')) return returnError('videoQuota') - if (result.videoQuotaDaily && !isUserVideoQuotaDailyValid(result.videoQuotaDaily + '')) return returnError('videoQuotaDaily') - - if (result.userUpdater && typeof result.userUpdater !== 'function') { - logger.error('Auth method %s of plugin %s did not provide a valid user updater function.', authName, npmName) - return false - } - - return true -} - -function buildUserResult (pluginResult: RegisterServerAuthenticatedResult) { - return { - username: pluginResult.username, - email: pluginResult.email, - role: pluginResult.role ?? UserRole.USER, - displayName: pluginResult.displayName || pluginResult.username, - - adminFlags: pluginResult.adminFlags ?? UserAdminFlag.NONE, - - videoQuota: pluginResult.videoQuota, - videoQuotaDaily: pluginResult.videoQuotaDaily - } -} - -// --------------------------------------------------------------------------- - -export { - onExternalUserAuthenticated, - getBypassFromExternalAuth, - getAuthNameFromRefreshGrant, - getBypassFromPasswordGrant -} diff --git a/server/lib/auth/oauth-model.ts b/server/lib/auth/oauth-model.ts deleted file mode 100644 index d3a5eccd5..000000000 --- a/server/lib/auth/oauth-model.ts +++ /dev/null @@ -1,294 +0,0 @@ -import express from 'express' -import { AccessDeniedError } from '@node-oauth/oauth2-server' -import { PluginManager } from '@server/lib/plugins/plugin-manager' -import { AccountModel } from '@server/models/account/account' -import { AuthenticatedResultUpdaterFieldName, RegisterServerAuthenticatedResult } from '@server/types' -import { MOAuthClient } from '@server/types/models' -import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' -import { MUser, MUserDefault } from '@server/types/models/user/user' -import { pick } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { OAuthClientModel } from '../../models/oauth/oauth-client' -import { OAuthTokenModel } from '../../models/oauth/oauth-token' -import { UserModel } from '../../models/user/user' -import { findAvailableLocalActorName } from '../local-actor' -import { buildUser, createUserAccountAndChannelAndPlaylist } from '../user' -import { ExternalUser } from './external-auth' -import { TokensCache } from './tokens-cache' - -type TokenInfo = { - accessToken: string - refreshToken: string - accessTokenExpiresAt: Date - refreshTokenExpiresAt: Date -} - -export type BypassLogin = { - bypass: boolean - pluginName: string - authName?: string - user: ExternalUser - userUpdater: RegisterServerAuthenticatedResult['userUpdater'] -} - -async function getAccessToken (bearerToken: string) { - logger.debug('Getting access token.') - - if (!bearerToken) return undefined - - let tokenModel: MOAuthTokenUser - - if (TokensCache.Instance.hasToken(bearerToken)) { - tokenModel = TokensCache.Instance.getByToken(bearerToken) - } else { - tokenModel = await OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) - - if (tokenModel) TokensCache.Instance.setToken(tokenModel) - } - - if (!tokenModel) return undefined - - if (tokenModel.User.pluginAuth) { - const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'access') - - if (valid !== true) return undefined - } - - return tokenModel -} - -function getClient (clientId: string, clientSecret: string) { - logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') - - return OAuthClientModel.getByIdAndSecret(clientId, clientSecret) -} - -async function getRefreshToken (refreshToken: string) { - logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') - - const tokenInfo = await OAuthTokenModel.getByRefreshTokenAndPopulateClient(refreshToken) - if (!tokenInfo) return undefined - - const tokenModel = tokenInfo.token - - if (tokenModel.User.pluginAuth) { - const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'refresh') - - if (valid !== true) return undefined - } - - return tokenInfo -} - -async function getUser (usernameOrEmail?: string, password?: string, bypassLogin?: BypassLogin) { - // Special treatment coming from a plugin - if (bypassLogin && bypassLogin.bypass === true) { - logger.info('Bypassing oauth login by plugin %s.', bypassLogin.pluginName) - - let user = await UserModel.loadByEmail(bypassLogin.user.email) - - if (!user) user = await createUserFromExternal(bypassLogin.pluginName, bypassLogin.user) - else user = await updateUserFromExternal(user, bypassLogin.user, bypassLogin.userUpdater) - - // Cannot create a user - if (!user) throw new AccessDeniedError('Cannot create such user: an actor with that name already exists.') - - // If the user does not belongs to a plugin, it was created before its installation - // Then we just go through a regular login process - if (user.pluginAuth !== null) { - // This user does not belong to this plugin, skip it - if (user.pluginAuth !== bypassLogin.pluginName) { - logger.info( - 'Cannot bypass oauth login by plugin %s because %s has another plugin auth method (%s).', - bypassLogin.pluginName, bypassLogin.user.email, user.pluginAuth - ) - - return null - } - - checkUserValidityOrThrow(user) - - return user - } - } - - logger.debug('Getting User (username/email: ' + usernameOrEmail + ', password: ******).') - - const user = await UserModel.loadByUsernameOrEmail(usernameOrEmail) - - // If we don't find the user, or if the user belongs to a plugin - if (!user || user.pluginAuth !== null || !password) return null - - const passwordMatch = await user.isPasswordMatch(password) - if (passwordMatch !== true) return null - - checkUserValidityOrThrow(user) - - if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION && user.emailVerified === false) { - throw new AccessDeniedError('User email is not verified.') - } - - return user -} - -async function revokeToken ( - tokenInfo: { refreshToken: string }, - options: { - req?: express.Request - explicitLogout?: boolean - } = {} -): Promise<{ success: boolean, redirectUrl?: string }> { - const { req, explicitLogout } = options - - const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) - - if (token) { - let redirectUrl: string - - if (explicitLogout === true && token.User.pluginAuth && token.authName) { - redirectUrl = await PluginManager.Instance.onLogout(token.User.pluginAuth, token.authName, token.User, req) - } - - TokensCache.Instance.clearCacheByToken(token.accessToken) - - token.destroy() - .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) - - return { success: true, redirectUrl } - } - - return { success: false } -} - -async function saveToken ( - token: TokenInfo, - client: MOAuthClient, - user: MUser, - options: { - refreshTokenAuthName?: string - bypassLogin?: BypassLogin - } = {} -) { - const { refreshTokenAuthName, bypassLogin } = options - let authName: string = null - - if (bypassLogin?.bypass === true) { - authName = bypassLogin.authName - } else if (refreshTokenAuthName) { - authName = refreshTokenAuthName - } - - logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') - - const tokenToCreate = { - accessToken: token.accessToken, - accessTokenExpiresAt: token.accessTokenExpiresAt, - refreshToken: token.refreshToken, - refreshTokenExpiresAt: token.refreshTokenExpiresAt, - authName, - oAuthClientId: client.id, - userId: user.id - } - - const tokenCreated = await OAuthTokenModel.create(tokenToCreate) - - user.lastLoginDate = new Date() - await user.save() - - return { - accessToken: tokenCreated.accessToken, - accessTokenExpiresAt: tokenCreated.accessTokenExpiresAt, - refreshToken: tokenCreated.refreshToken, - refreshTokenExpiresAt: tokenCreated.refreshTokenExpiresAt, - client, - user, - accessTokenExpiresIn: buildExpiresIn(tokenCreated.accessTokenExpiresAt), - refreshTokenExpiresIn: buildExpiresIn(tokenCreated.refreshTokenExpiresAt) - } -} - -export { - getAccessToken, - getClient, - getRefreshToken, - getUser, - revokeToken, - saveToken -} - -// --------------------------------------------------------------------------- - -async function createUserFromExternal (pluginAuth: string, userOptions: ExternalUser) { - const username = await findAvailableLocalActorName(userOptions.username) - - const userToCreate = buildUser({ - ...pick(userOptions, [ 'email', 'role', 'adminFlags', 'videoQuota', 'videoQuotaDaily' ]), - - username, - emailVerified: null, - password: null, - pluginAuth - }) - - const { user } = await createUserAccountAndChannelAndPlaylist({ - userToCreate, - userDisplayName: userOptions.displayName - }) - - return user -} - -async function updateUserFromExternal ( - user: MUserDefault, - userOptions: ExternalUser, - userUpdater: RegisterServerAuthenticatedResult['userUpdater'] -) { - if (!userUpdater) return user - - { - type UserAttributeKeys = keyof AttributesOnly - const mappingKeys: { [ id in UserAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { - role: 'role', - adminFlags: 'adminFlags', - videoQuota: 'videoQuota', - videoQuotaDaily: 'videoQuotaDaily' - } - - for (const modelKey of Object.keys(mappingKeys)) { - const pluginOptionKey = mappingKeys[modelKey] - - const newValue = userUpdater({ fieldName: pluginOptionKey, currentValue: user[modelKey], newValue: userOptions[pluginOptionKey] }) - user.set(modelKey, newValue) - } - } - - { - type AccountAttributeKeys = keyof Partial> - const mappingKeys: { [ id in AccountAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { - name: 'displayName' - } - - for (const modelKey of Object.keys(mappingKeys)) { - const optionKey = mappingKeys[modelKey] - - const newValue = userUpdater({ fieldName: optionKey, currentValue: user.Account[modelKey], newValue: userOptions[optionKey] }) - user.Account.set(modelKey, newValue) - } - } - - logger.debug('Updated user %s with plugin userUpdated function.', user.email, { user, userOptions }) - - user.Account = await user.Account.save() - - return user.save() -} - -function checkUserValidityOrThrow (user: MUser) { - if (user.blocked) throw new AccessDeniedError('User is blocked.') -} - -function buildExpiresIn (expiresAt: Date) { - return Math.floor((expiresAt.getTime() - new Date().getTime()) / 1000) -} diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts deleted file mode 100644 index 887c4f7c9..000000000 --- a/server/lib/auth/oauth.ts +++ /dev/null @@ -1,223 +0,0 @@ -import express from 'express' -import OAuth2Server, { - InvalidClientError, - InvalidGrantError, - InvalidRequestError, - Request, - Response, - UnauthorizedClientError, - UnsupportedGrantTypeError -} from '@node-oauth/oauth2-server' -import { randomBytesPromise } from '@server/helpers/core-utils' -import { isOTPValid } from '@server/helpers/otp' -import { CONFIG } from '@server/initializers/config' -import { UserRegistrationModel } from '@server/models/user/user-registration' -import { MOAuthClient } from '@server/types/models' -import { sha1 } from '@shared/extra-utils' -import { HttpStatusCode, ServerErrorCode, UserRegistrationState } from '@shared/models' -import { OTP } from '../../initializers/constants' -import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' - -class MissingTwoFactorError extends Error { - code = HttpStatusCode.UNAUTHORIZED_401 - name = ServerErrorCode.MISSING_TWO_FACTOR -} - -class InvalidTwoFactorError extends Error { - code = HttpStatusCode.BAD_REQUEST_400 - name = ServerErrorCode.INVALID_TWO_FACTOR -} - -class RegistrationWaitingForApproval extends Error { - code = HttpStatusCode.BAD_REQUEST_400 - name = ServerErrorCode.ACCOUNT_WAITING_FOR_APPROVAL -} - -class RegistrationApprovalRejected extends Error { - code = HttpStatusCode.BAD_REQUEST_400 - name = ServerErrorCode.ACCOUNT_APPROVAL_REJECTED -} - -/** - * - * Reimplement some functions of OAuth2Server to inject external auth methods - * - */ -const oAuthServer = new OAuth2Server({ - // Wants seconds - accessTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN / 1000, - refreshTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN / 1000, - - // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications - model: require('./oauth-model') -}) - -// --------------------------------------------------------------------------- - -async function handleOAuthToken (req: express.Request, options: { refreshTokenAuthName?: string, bypassLogin?: BypassLogin }) { - const request = new Request(req) - const { refreshTokenAuthName, bypassLogin } = options - - if (request.method !== 'POST') { - throw new InvalidRequestError('Invalid request: method must be POST') - } - - if (!request.is([ 'application/x-www-form-urlencoded' ])) { - throw new InvalidRequestError('Invalid request: content must be application/x-www-form-urlencoded') - } - - const clientId = request.body.client_id - const clientSecret = request.body.client_secret - - if (!clientId || !clientSecret) { - throw new InvalidClientError('Invalid client: cannot retrieve client credentials') - } - - const client = await getClient(clientId, clientSecret) - if (!client) { - throw new InvalidClientError('Invalid client: client is invalid') - } - - const grantType = request.body.grant_type - if (!grantType) { - throw new InvalidRequestError('Missing parameter: `grant_type`') - } - - if (![ 'password', 'refresh_token' ].includes(grantType)) { - throw new UnsupportedGrantTypeError('Unsupported grant type: `grant_type` is invalid') - } - - if (!client.grants.includes(grantType)) { - throw new UnauthorizedClientError('Unauthorized client: `grant_type` is invalid') - } - - if (grantType === 'password') { - return handlePasswordGrant({ - request, - client, - bypassLogin - }) - } - - return handleRefreshGrant({ - request, - client, - refreshTokenAuthName - }) -} - -function handleOAuthAuthenticate ( - req: express.Request, - res: express.Response -) { - return oAuthServer.authenticate(new Request(req), new Response(res)) -} - -export { - MissingTwoFactorError, - InvalidTwoFactorError, - - handleOAuthToken, - handleOAuthAuthenticate -} - -// --------------------------------------------------------------------------- - -async function handlePasswordGrant (options: { - request: Request - client: MOAuthClient - bypassLogin?: BypassLogin -}) { - const { request, client, bypassLogin } = options - - if (!request.body.username) { - throw new InvalidRequestError('Missing parameter: `username`') - } - - if (!bypassLogin && !request.body.password) { - throw new InvalidRequestError('Missing parameter: `password`') - } - - const user = await getUser(request.body.username, request.body.password, bypassLogin) - if (!user) { - const registration = await UserRegistrationModel.loadByEmailOrUsername(request.body.username) - - if (registration?.state === UserRegistrationState.REJECTED) { - throw new RegistrationApprovalRejected('Registration approval for this account has been rejected') - } else if (registration?.state === UserRegistrationState.PENDING) { - throw new RegistrationWaitingForApproval('Registration for this account is awaiting approval') - } - - throw new InvalidGrantError('Invalid grant: user credentials are invalid') - } - - if (user.otpSecret) { - if (!request.headers[OTP.HEADER_NAME]) { - throw new MissingTwoFactorError('Missing two factor header') - } - - if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) { - throw new InvalidTwoFactorError('Invalid two factor header') - } - } - - const token = await buildToken() - - return saveToken(token, client, user, { bypassLogin }) -} - -async function handleRefreshGrant (options: { - request: Request - client: MOAuthClient - refreshTokenAuthName: string -}) { - const { request, client, refreshTokenAuthName } = options - - if (!request.body.refresh_token) { - throw new InvalidRequestError('Missing parameter: `refresh_token`') - } - - const refreshToken = await getRefreshToken(request.body.refresh_token) - - if (!refreshToken) { - throw new InvalidGrantError('Invalid grant: refresh token is invalid') - } - - if (refreshToken.client.id !== client.id) { - throw new InvalidGrantError('Invalid grant: refresh token is invalid') - } - - if (refreshToken.refreshTokenExpiresAt && refreshToken.refreshTokenExpiresAt < new Date()) { - throw new InvalidGrantError('Invalid grant: refresh token has expired') - } - - await revokeToken({ refreshToken: refreshToken.refreshToken }) - - const token = await buildToken() - - return saveToken(token, client, refreshToken.user, { refreshTokenAuthName }) -} - -function generateRandomToken () { - return randomBytesPromise(256) - .then(buffer => sha1(buffer)) -} - -function getTokenExpiresAt (type: 'access' | 'refresh') { - const lifetime = type === 'access' - ? CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN - : CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN - - return new Date(Date.now() + lifetime) -} - -async function buildToken () { - const [ accessToken, refreshToken ] = await Promise.all([ generateRandomToken(), generateRandomToken() ]) - - return { - accessToken, - refreshToken, - accessTokenExpiresAt: getTokenExpiresAt('access'), - refreshTokenExpiresAt: getTokenExpiresAt('refresh') - } -} diff --git a/server/lib/auth/tokens-cache.ts b/server/lib/auth/tokens-cache.ts deleted file mode 100644 index e7b12159b..000000000 --- a/server/lib/auth/tokens-cache.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { LRUCache } from 'lru-cache' -import { MOAuthTokenUser } from '@server/types/models' -import { LRU_CACHE } from '../../initializers/constants' - -export class TokensCache { - - private static instance: TokensCache - - private readonly accessTokenCache = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) - private readonly userHavingToken = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) - - private constructor () { } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - hasToken (token: string) { - return this.accessTokenCache.has(token) - } - - getByToken (token: string) { - return this.accessTokenCache.get(token) - } - - setToken (token: MOAuthTokenUser) { - this.accessTokenCache.set(token.accessToken, token) - this.userHavingToken.set(token.userId, token.accessToken) - } - - deleteUserToken (userId: number) { - this.clearCacheByUserId(userId) - } - - clearCacheByUserId (userId: number) { - const token = this.userHavingToken.get(userId) - - if (token !== undefined) { - this.accessTokenCache.delete(token) - this.userHavingToken.delete(userId) - } - } - - clearCacheByToken (token: string) { - const tokenModel = this.accessTokenCache.get(token) - - if (tokenModel !== undefined) { - this.userHavingToken.delete(tokenModel.userId) - this.accessTokenCache.delete(token) - } - } -} diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts deleted file mode 100644 index 009e229ce..000000000 --- a/server/lib/blocklist.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { sequelizeTypescript } from '@server/initializers/database' -import { getServerActor } from '@server/models/application/application' -import { MAccountBlocklist, MAccountId, MAccountHost, MServerBlocklist } from '@server/types/models' -import { AccountBlocklistModel } from '../models/account/account-blocklist' -import { ServerBlocklistModel } from '../models/server/server-blocklist' - -function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { - return sequelizeTypescript.transaction(async t => { - return AccountBlocklistModel.upsert({ - accountId: byAccountId, - targetAccountId - }, { transaction: t }) - }) -} - -function addServerInBlocklist (byAccountId: number, targetServerId: number) { - return sequelizeTypescript.transaction(async t => { - return ServerBlocklistModel.upsert({ - accountId: byAccountId, - targetServerId - }, { transaction: t }) - }) -} - -function removeAccountFromBlocklist (accountBlock: MAccountBlocklist) { - return sequelizeTypescript.transaction(async t => { - return accountBlock.destroy({ transaction: t }) - }) -} - -function removeServerFromBlocklist (serverBlock: MServerBlocklist) { - return sequelizeTypescript.transaction(async t => { - return serverBlock.destroy({ transaction: t }) - }) -} - -async function isBlockedByServerOrAccount (targetAccount: MAccountHost, userAccount?: MAccountId) { - const serverAccountId = (await getServerActor()).Account.id - const sourceAccounts = [ serverAccountId ] - - if (userAccount) sourceAccounts.push(userAccount.id) - - const accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, targetAccount.id) - if (accountMutedHash[serverAccountId] || (userAccount && accountMutedHash[userAccount.id])) { - return true - } - - const instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, targetAccount.Actor.serverId) - if (instanceMutedHash[serverAccountId] || (userAccount && instanceMutedHash[userAccount.id])) { - return true - } - - return false -} - -export { - addAccountInBlocklist, - addServerInBlocklist, - removeAccountFromBlocklist, - removeServerFromBlocklist, - isBlockedByServerOrAccount -} diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts deleted file mode 100644 index 8e0c9e328..000000000 --- a/server/lib/client-html.ts +++ /dev/null @@ -1,623 +0,0 @@ -import express from 'express' -import { pathExists, readFile } from 'fs-extra' -import { truncate } from 'lodash' -import { join } from 'path' -import validator from 'validator' -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { toCompleteUUID } from '@server/helpers/custom-validators/misc' -import { mdToOneLinePlainText } from '@server/helpers/markdown' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { root } from '@shared/core-utils' -import { escapeHTML } from '@shared/core-utils/renderer' -import { sha256 } from '@shared/extra-utils' -import { HTMLServerConfig } from '@shared/models' -import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' -import { HttpStatusCode } from '../../shared/models/http/http-error-codes' -import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' -import { logger } from '../helpers/logger' -import { CONFIG } from '../initializers/config' -import { - ACCEPT_HEADERS, - CUSTOM_HTML_TAG_COMMENTS, - EMBED_SIZE, - FILES_CONTENT_HASH, - PLUGIN_GLOBAL_CSS_PATH, - WEBSERVER -} from '../initializers/constants' -import { AccountModel } from '../models/account/account' -import { VideoModel } from '../models/video/video' -import { VideoChannelModel } from '../models/video/video-channel' -import { VideoPlaylistModel } from '../models/video/video-playlist' -import { MAccountHost, MChannelHost, MVideo, MVideoPlaylist } from '../types/models' -import { getActivityStreamDuration } from './activitypub/activity' -import { getBiggestActorImage } from './actor-image' -import { Hooks } from './plugins/hooks' -import { ServerConfigManager } from './server-config-manager' -import { isVideoInPrivateDirectory } from './video-privacy' - -type Tags = { - ogType: string - twitterCard: 'player' | 'summary' | 'summary_large_image' - schemaType: string - - list?: { - numberOfItems: number - } - - escapedSiteName: string - escapedTitle: string - escapedTruncatedDescription: string - - url: string - originUrl: string - - disallowIndexation?: boolean - - embed?: { - url: string - createdAt: string - duration?: string - views?: number - } - - image: { - url: string - width?: number - height?: number - } -} - -type HookContext = { - video?: MVideo - playlist?: MVideoPlaylist -} - -class ClientHtml { - - private static htmlCache: { [path: string]: string } = {} - - static invalidCache () { - logger.info('Cleaning HTML cache.') - - ClientHtml.htmlCache = {} - } - - static async getDefaultHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { - const html = paramLang - ? await ClientHtml.getIndexHTML(req, res, paramLang) - : await ClientHtml.getIndexHTML(req, res) - - let customHtml = ClientHtml.addTitleTag(html) - customHtml = ClientHtml.addDescriptionTag(customHtml) - - return customHtml - } - - static async getWatchHTMLPage (videoIdArg: string, req: express.Request, res: express.Response) { - const videoId = toCompleteUUID(videoIdArg) - - // Let Angular application handle errors - if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) { - res.status(HttpStatusCode.NOT_FOUND_404) - return ClientHtml.getIndexHTML(req, res) - } - - const [ html, video ] = await Promise.all([ - ClientHtml.getIndexHTML(req, res), - VideoModel.loadWithBlacklist(videoId) - ]) - - // Let Angular application handle errors - if (!video || isVideoInPrivateDirectory(video.privacy) || video.VideoBlacklist) { - res.status(HttpStatusCode.NOT_FOUND_404) - return html - } - const escapedTruncatedDescription = buildEscapedTruncatedDescription(video.description) - - let customHtml = ClientHtml.addTitleTag(html, video.name) - customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) - - const url = WEBSERVER.URL + video.getWatchStaticPath() - const originUrl = video.url - const title = video.name - const siteName = CONFIG.INSTANCE.NAME - - const image = { - url: WEBSERVER.URL + video.getPreviewStaticPath() - } - - const embed = { - url: WEBSERVER.URL + video.getEmbedStaticPath(), - createdAt: video.createdAt.toISOString(), - duration: getActivityStreamDuration(video.duration), - views: video.views - } - - const ogType = 'video' - const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary_large_image' - const schemaType = 'VideoObject' - - customHtml = await ClientHtml.addTags(customHtml, { - url, - originUrl, - escapedSiteName: escapeHTML(siteName), - escapedTitle: escapeHTML(title), - escapedTruncatedDescription, - disallowIndexation: video.privacy !== VideoPrivacy.PUBLIC, - image, - embed, - ogType, - twitterCard, - schemaType - }, { video }) - - return customHtml - } - - static async getWatchPlaylistHTMLPage (videoPlaylistIdArg: string, req: express.Request, res: express.Response) { - const videoPlaylistId = toCompleteUUID(videoPlaylistIdArg) - - // Let Angular application handle errors - if (!validator.isInt(videoPlaylistId) && !validator.isUUID(videoPlaylistId, 4)) { - res.status(HttpStatusCode.NOT_FOUND_404) - return ClientHtml.getIndexHTML(req, res) - } - - const [ html, videoPlaylist ] = await Promise.all([ - ClientHtml.getIndexHTML(req, res), - VideoPlaylistModel.loadWithAccountAndChannel(videoPlaylistId, null) - ]) - - // Let Angular application handle errors - if (!videoPlaylist || videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { - res.status(HttpStatusCode.NOT_FOUND_404) - return html - } - - const escapedTruncatedDescription = buildEscapedTruncatedDescription(videoPlaylist.description) - - let customHtml = ClientHtml.addTitleTag(html, videoPlaylist.name) - customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) - - const url = WEBSERVER.URL + videoPlaylist.getWatchStaticPath() - const originUrl = videoPlaylist.url - const title = videoPlaylist.name - const siteName = CONFIG.INSTANCE.NAME - - const image = { - url: videoPlaylist.getThumbnailUrl() - } - - const embed = { - url: WEBSERVER.URL + videoPlaylist.getEmbedStaticPath(), - createdAt: videoPlaylist.createdAt.toISOString() - } - - const list = { - numberOfItems: videoPlaylist.get('videosLength') as number - } - - const ogType = 'video' - const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary' - const schemaType = 'ItemList' - - customHtml = await ClientHtml.addTags(customHtml, { - url, - originUrl, - escapedSiteName: escapeHTML(siteName), - escapedTitle: escapeHTML(title), - escapedTruncatedDescription, - disallowIndexation: videoPlaylist.privacy !== VideoPlaylistPrivacy.PUBLIC, - embed, - image, - list, - ogType, - twitterCard, - schemaType - }, { playlist: videoPlaylist }) - - return customHtml - } - - static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { - const accountModelPromise = AccountModel.loadByNameWithHost(nameWithHost) - return this.getAccountOrChannelHTMLPage(() => accountModelPromise, req, res) - } - - static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { - const videoChannelModelPromise = VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) - return this.getAccountOrChannelHTMLPage(() => videoChannelModelPromise, req, res) - } - - static async getActorHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { - const [ account, channel ] = await Promise.all([ - AccountModel.loadByNameWithHost(nameWithHost), - VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) - ]) - - return this.getAccountOrChannelHTMLPage(() => Promise.resolve(account || channel), req, res) - } - - static async getEmbedHTML () { - const path = ClientHtml.getEmbedPath() - - // Disable HTML cache in dev mode because webpack can regenerate JS files - if (!isTestOrDevInstance() && ClientHtml.htmlCache[path]) { - return ClientHtml.htmlCache[path] - } - - const buffer = await readFile(path) - const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() - - let html = buffer.toString() - html = await ClientHtml.addAsyncPluginCSS(html) - html = ClientHtml.addCustomCSS(html) - html = ClientHtml.addTitleTag(html) - html = ClientHtml.addDescriptionTag(html) - html = ClientHtml.addServerConfig(html, serverConfig) - - ClientHtml.htmlCache[path] = html - - return html - } - - private static async getAccountOrChannelHTMLPage ( - loader: () => Promise, - req: express.Request, - res: express.Response - ) { - const [ html, entity ] = await Promise.all([ - ClientHtml.getIndexHTML(req, res), - loader() - ]) - - // Let Angular application handle errors - if (!entity) { - res.status(HttpStatusCode.NOT_FOUND_404) - return ClientHtml.getIndexHTML(req, res) - } - - const escapedTruncatedDescription = buildEscapedTruncatedDescription(entity.description) - - let customHtml = ClientHtml.addTitleTag(html, entity.getDisplayName()) - customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) - - const url = entity.getClientUrl() - const originUrl = entity.Actor.url - const siteName = CONFIG.INSTANCE.NAME - const title = entity.getDisplayName() - - const avatar = getBiggestActorImage(entity.Actor.Avatars) - const image = { - url: ActorImageModel.getImageUrl(avatar), - width: avatar?.width, - height: avatar?.height - } - - const ogType = 'website' - const twitterCard = 'summary' - const schemaType = 'ProfilePage' - - customHtml = await ClientHtml.addTags(customHtml, { - url, - originUrl, - escapedTitle: escapeHTML(title), - escapedSiteName: escapeHTML(siteName), - escapedTruncatedDescription, - image, - ogType, - twitterCard, - schemaType, - disallowIndexation: !entity.Actor.isOwned() - }, {}) - - return customHtml - } - - private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { - const path = ClientHtml.getIndexPath(req, res, paramLang) - if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] - - const buffer = await readFile(path) - const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() - - let html = buffer.toString() - - html = ClientHtml.addManifestContentHash(html) - html = ClientHtml.addFaviconContentHash(html) - html = ClientHtml.addLogoContentHash(html) - html = ClientHtml.addCustomCSS(html) - html = ClientHtml.addServerConfig(html, serverConfig) - html = await ClientHtml.addAsyncPluginCSS(html) - - ClientHtml.htmlCache[path] = html - - return html - } - - private static getIndexPath (req: express.Request, res: express.Response, paramLang: string) { - let lang: string - - // Check param lang validity - if (paramLang && is18nLocale(paramLang)) { - lang = paramLang - - // Save locale in cookies - res.cookie('clientLanguage', lang, { - secure: WEBSERVER.SCHEME === 'https', - sameSite: 'none', - maxAge: 1000 * 3600 * 24 * 90 // 3 months - }) - - } else if (req.cookies.clientLanguage && is18nLocale(req.cookies.clientLanguage)) { - lang = req.cookies.clientLanguage - } else { - lang = req.acceptsLanguages(POSSIBLE_LOCALES) || getDefaultLocale() - } - - logger.debug( - 'Serving %s HTML language', buildFileLocale(lang), - { cookie: req.cookies?.clientLanguage, paramLang, acceptLanguage: req.headers['accept-language'] } - ) - - return join(root(), 'client', 'dist', buildFileLocale(lang), 'index.html') - } - - private static getEmbedPath () { - return join(root(), 'client', 'dist', 'standalone', 'videos', 'embed.html') - } - - private static addManifestContentHash (htmlStringPage: string) { - return htmlStringPage.replace('[manifestContentHash]', FILES_CONTENT_HASH.MANIFEST) - } - - private static addFaviconContentHash (htmlStringPage: string) { - return htmlStringPage.replace('[faviconContentHash]', FILES_CONTENT_HASH.FAVICON) - } - - private static addLogoContentHash (htmlStringPage: string) { - return htmlStringPage.replace('[logoContentHash]', FILES_CONTENT_HASH.LOGO) - } - - private static addTitleTag (htmlStringPage: string, title?: string) { - let text = title || CONFIG.INSTANCE.NAME - if (title) text += ` - ${CONFIG.INSTANCE.NAME}` - - const titleTag = `${escapeHTML(text)}` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) - } - - private static addDescriptionTag (htmlStringPage: string, escapedTruncatedDescription?: string) { - const content = escapedTruncatedDescription || escapeHTML(CONFIG.INSTANCE.SHORT_DESCRIPTION) - const descriptionTag = `` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) - } - - private static addCustomCSS (htmlStringPage: string) { - const styleTag = `` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) - } - - private static addServerConfig (htmlStringPage: string, serverConfig: HTMLServerConfig) { - // Stringify the JSON object, and then stringify the string object so we can inject it into the HTML - const serverConfigString = JSON.stringify(JSON.stringify(serverConfig)) - const configScriptTag = `` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.SERVER_CONFIG, configScriptTag) - } - - private static async addAsyncPluginCSS (htmlStringPage: string) { - if (!pathExists(PLUGIN_GLOBAL_CSS_PATH)) { - logger.info('Plugin Global CSS file is not available (generation may still be in progress), ignoring it.') - return htmlStringPage - } - - let globalCSSContent: Buffer - - try { - globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH) - } catch (err) { - logger.error('Error retrieving the Plugin Global CSS file, ignoring it.', { err }) - return htmlStringPage - } - - if (globalCSSContent.byteLength === 0) return htmlStringPage - - const fileHash = sha256(globalCSSContent) - const linkTag = `` - - return htmlStringPage.replace('', linkTag + '') - } - - private static generateOpenGraphMetaTags (tags: Tags) { - const metaTags = { - 'og:type': tags.ogType, - 'og:site_name': tags.escapedSiteName, - 'og:title': tags.escapedTitle, - 'og:image': tags.image.url - } - - if (tags.image.width && tags.image.height) { - metaTags['og:image:width'] = tags.image.width - metaTags['og:image:height'] = tags.image.height - } - - metaTags['og:url'] = tags.url - metaTags['og:description'] = tags.escapedTruncatedDescription - - if (tags.embed) { - metaTags['og:video:url'] = tags.embed.url - metaTags['og:video:secure_url'] = tags.embed.url - metaTags['og:video:type'] = 'text/html' - metaTags['og:video:width'] = EMBED_SIZE.width - metaTags['og:video:height'] = EMBED_SIZE.height - } - - return metaTags - } - - private static generateStandardMetaTags (tags: Tags) { - return { - name: tags.escapedTitle, - description: tags.escapedTruncatedDescription, - image: tags.image.url - } - } - - private static generateTwitterCardMetaTags (tags: Tags) { - const metaTags = { - 'twitter:card': tags.twitterCard, - 'twitter:site': CONFIG.SERVICES.TWITTER.USERNAME, - 'twitter:title': tags.escapedTitle, - 'twitter:description': tags.escapedTruncatedDescription, - 'twitter:image': tags.image.url - } - - if (tags.image.width && tags.image.height) { - metaTags['twitter:image:width'] = tags.image.width - metaTags['twitter:image:height'] = tags.image.height - } - - if (tags.twitterCard === 'player') { - metaTags['twitter:player'] = tags.embed.url - metaTags['twitter:player:width'] = EMBED_SIZE.width - metaTags['twitter:player:height'] = EMBED_SIZE.height - } - - return metaTags - } - - private static async generateSchemaTags (tags: Tags, context: HookContext) { - const schema = { - '@context': 'http://schema.org', - '@type': tags.schemaType, - 'name': tags.escapedTitle, - 'description': tags.escapedTruncatedDescription, - 'image': tags.image.url, - 'url': tags.url - } - - if (tags.list) { - schema['numberOfItems'] = tags.list.numberOfItems - schema['thumbnailUrl'] = tags.image.url - } - - if (tags.embed) { - schema['embedUrl'] = tags.embed.url - schema['uploadDate'] = tags.embed.createdAt - - if (tags.embed.duration) schema['duration'] = tags.embed.duration - - schema['thumbnailUrl'] = tags.image.url - schema['contentUrl'] = tags.url - } - - return Hooks.wrapObject(schema, 'filter:html.client.json-ld.result', context) - } - - private static async addTags (htmlStringPage: string, tagsValues: Tags, context: HookContext) { - const openGraphMetaTags = this.generateOpenGraphMetaTags(tagsValues) - const standardMetaTags = this.generateStandardMetaTags(tagsValues) - const twitterCardMetaTags = this.generateTwitterCardMetaTags(tagsValues) - const schemaTags = await this.generateSchemaTags(tagsValues, context) - - const { url, escapedTitle, embed, originUrl, disallowIndexation } = tagsValues - - const oembedLinkTags: { type: string, href: string, escapedTitle: string }[] = [] - - if (embed) { - oembedLinkTags.push({ - type: 'application/json+oembed', - href: WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(url), - escapedTitle - }) - } - - let tagsStr = '' - - // Opengraph - Object.keys(openGraphMetaTags).forEach(tagName => { - const tagValue = openGraphMetaTags[tagName] - - tagsStr += `` - }) - - // Standard - Object.keys(standardMetaTags).forEach(tagName => { - const tagValue = standardMetaTags[tagName] - - tagsStr += `` - }) - - // Twitter card - Object.keys(twitterCardMetaTags).forEach(tagName => { - const tagValue = twitterCardMetaTags[tagName] - - tagsStr += `` - }) - - // OEmbed - for (const oembedLinkTag of oembedLinkTags) { - tagsStr += `` - } - - // Schema.org - if (schemaTags) { - tagsStr += `` - } - - // SEO, use origin URL - tagsStr += `` - - if (disallowIndexation) { - tagsStr += `` - } - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.META_TAGS, tagsStr) - } -} - -function sendHTML (html: string, res: express.Response, localizedHTML: boolean = false) { - res.set('Content-Type', 'text/html; charset=UTF-8') - - if (localizedHTML) { - res.set('Vary', 'Accept-Language') - } - - return res.send(html) -} - -async function serveIndexHTML (req: express.Request, res: express.Response) { - if (req.accepts(ACCEPT_HEADERS) === 'html' || !req.headers.accept) { - try { - await generateHTMLPage(req, res, req.params.language) - return - } catch (err) { - logger.error('Cannot generate HTML page.', { err }) - return res.status(HttpStatusCode.INTERNAL_SERVER_ERROR_500).end() - } - } - - return res.status(HttpStatusCode.NOT_ACCEPTABLE_406).end() -} - -// --------------------------------------------------------------------------- - -export { - ClientHtml, - sendHTML, - serveIndexHTML -} - -async function generateHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { - const html = await ClientHtml.getDefaultHTMLPage(req, res, paramLang) - - return sendHTML(html, res, true) -} - -function buildEscapedTruncatedDescription (description: string) { - return truncate(mdToOneLinePlainText(description), { length: 200 }) -} diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts deleted file mode 100644 index f5c3e4745..000000000 --- a/server/lib/emailer.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { readFileSync } from 'fs-extra' -import { merge } from 'lodash' -import { createTransport, Transporter } from 'nodemailer' -import { join } from 'path' -import { arrayify, root } from '@shared/core-utils' -import { EmailPayload, UserRegistrationState } from '@shared/models' -import { SendEmailDefaultOptions } from '../../shared/models/server/emailer.model' -import { isTestOrDevInstance } from '../helpers/core-utils' -import { bunyanLogger, logger } from '../helpers/logger' -import { CONFIG, isEmailEnabled } from '../initializers/config' -import { WEBSERVER } from '../initializers/constants' -import { MRegistration, MUser } from '../types/models' -import { JobQueue } from './job-queue' - -const Email = require('email-templates') - -class Emailer { - - private static instance: Emailer - private initialized = false - private transporter: Transporter - - private constructor () { - } - - init () { - // Already initialized - if (this.initialized === true) return - this.initialized = true - - if (!isEmailEnabled()) { - if (!isTestOrDevInstance()) { - logger.error('Cannot use SMTP server because of lack of configuration. PeerTube will not be able to send mails!') - } - - return - } - - if (CONFIG.SMTP.TRANSPORT === 'smtp') this.initSMTPTransport() - else if (CONFIG.SMTP.TRANSPORT === 'sendmail') this.initSendmailTransport() - } - - async checkConnection () { - if (!this.transporter || CONFIG.SMTP.TRANSPORT !== 'smtp') return - - logger.info('Testing SMTP server...') - - try { - const success = await this.transporter.verify() - if (success !== true) this.warnOnConnectionFailure() - - logger.info('Successfully connected to SMTP server.') - } catch (err) { - this.warnOnConnectionFailure(err) - } - } - - addPasswordResetEmailJob (username: string, to: string, resetPasswordUrl: string) { - const emailPayload: EmailPayload = { - template: 'password-reset', - to: [ to ], - subject: 'Reset your account password', - locals: { - username, - resetPasswordUrl, - - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addPasswordCreateEmailJob (username: string, to: string, createPasswordUrl: string) { - const emailPayload: EmailPayload = { - template: 'password-create', - to: [ to ], - subject: 'Create your account password', - locals: { - username, - createPasswordUrl, - - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addVerifyEmailJob (options: { - username: string - isRegistrationRequest: boolean - to: string - verifyEmailUrl: string - }) { - const { username, isRegistrationRequest, to, verifyEmailUrl } = options - - const emailPayload: EmailPayload = { - template: 'verify-email', - to: [ to ], - subject: `Verify your email on ${CONFIG.INSTANCE.NAME}`, - locals: { - username, - verifyEmailUrl, - isRegistrationRequest, - - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addUserBlockJob (user: MUser, blocked: boolean, reason?: string) { - const reasonString = reason ? ` for the following reason: ${reason}` : '' - const blockedWord = blocked ? 'blocked' : 'unblocked' - - const to = user.email - const emailPayload: EmailPayload = { - to: [ to ], - subject: 'Account ' + blockedWord, - text: `Your account ${user.username} on ${CONFIG.INSTANCE.NAME} has been ${blockedWord}${reasonString}.` - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addContactFormJob (fromEmail: string, fromName: string, subject: string, body: string) { - const emailPayload: EmailPayload = { - template: 'contact-form', - to: [ CONFIG.ADMIN.EMAIL ], - replyTo: `"${fromName}" <${fromEmail}>`, - subject: `(contact form) ${subject}`, - locals: { - fromName, - fromEmail, - body, - - // There are not notification preferences for the contact form - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addUserRegistrationRequestProcessedJob (registration: MRegistration) { - let template: string - let subject: string - if (registration.state === UserRegistrationState.ACCEPTED) { - template = 'user-registration-request-accepted' - subject = `Your registration request for ${registration.username} has been accepted` - } else { - template = 'user-registration-request-rejected' - subject = `Your registration request for ${registration.username} has been rejected` - } - - const to = registration.email - const emailPayload: EmailPayload = { - to: [ to ], - template, - subject, - locals: { - username: registration.username, - moderationResponse: registration.moderationResponse, - loginLink: WEBSERVER.URL + '/login' - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - async sendMail (options: EmailPayload) { - if (!isEmailEnabled()) { - logger.info('Cannot send mail because SMTP is not configured.') - return - } - - const fromDisplayName = options.from - ? options.from - : CONFIG.INSTANCE.NAME - - const email = new Email({ - send: true, - htmlToText: { - selectors: [ - { selector: 'img', format: 'skip' }, - { selector: 'a', options: { hideLinkHrefIfSameAsText: true } } - ] - }, - message: { - from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>` - }, - transport: this.transporter, - views: { - root: join(root(), 'dist', 'server', 'lib', 'emails') - }, - subjectPrefix: CONFIG.EMAIL.SUBJECT.PREFIX - }) - - const toEmails = arrayify(options.to) - - for (const to of toEmails) { - const baseOptions: SendEmailDefaultOptions = { - template: 'common', - message: { - to, - from: options.from, - subject: options.subject, - replyTo: options.replyTo - }, - locals: { // default variables available in all templates - WEBSERVER, - EMAIL: CONFIG.EMAIL, - instanceName: CONFIG.INSTANCE.NAME, - text: options.text, - subject: options.subject - } - } - - // overridden/new variables given for a specific template in the payload - const sendOptions = merge(baseOptions, options) - - await email.send(sendOptions) - .then(res => logger.debug('Sent email.', { res })) - .catch(err => logger.error('Error in email sender.', { err })) - } - } - - private warnOnConnectionFailure (err?: Error) { - logger.error('Failed to connect to SMTP %s:%d.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT, { err }) - } - - private initSMTPTransport () { - logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) - - let tls - if (CONFIG.SMTP.CA_FILE) { - tls = { - ca: [ readFileSync(CONFIG.SMTP.CA_FILE) ] - } - } - - let auth - if (CONFIG.SMTP.USERNAME && CONFIG.SMTP.PASSWORD) { - auth = { - user: CONFIG.SMTP.USERNAME, - pass: CONFIG.SMTP.PASSWORD - } - } - - this.transporter = createTransport({ - host: CONFIG.SMTP.HOSTNAME, - port: CONFIG.SMTP.PORT, - secure: CONFIG.SMTP.TLS, - debug: CONFIG.LOG.LEVEL === 'debug', - logger: bunyanLogger as any, - ignoreTLS: CONFIG.SMTP.DISABLE_STARTTLS, - tls, - auth - }) - } - - private initSendmailTransport () { - logger.info('Using sendmail to send emails') - - this.transporter = createTransport({ - sendmail: true, - newline: 'unix', - path: CONFIG.SMTP.SENDMAIL, - logger: bunyanLogger - }) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - Emailer -} diff --git a/server/lib/emails/abuse-new-message/html.pug b/server/lib/emails/abuse-new-message/html.pug deleted file mode 100644 index c1d452e7d..000000000 --- a/server/lib/emails/abuse-new-message/html.pug +++ /dev/null @@ -1,11 +0,0 @@ -extends ../common/greetings -include ../common/mixins.pug - -block title - | New message on abuse report - -block content - p - | A new message by #{messageAccountName} was posted on #[a(href=abuseUrl) abuse report ##{abuseId}] on #{instanceName} - blockquote #{messageText} - br(style="display: none;") diff --git a/server/lib/emails/abuse-state-change/html.pug b/server/lib/emails/abuse-state-change/html.pug deleted file mode 100644 index bb243e729..000000000 --- a/server/lib/emails/abuse-state-change/html.pug +++ /dev/null @@ -1,9 +0,0 @@ -extends ../common/greetings -include ../common/mixins.pug - -block title - | Abuse report state changed - -block content - p - | #[a(href=abuseUrl) Your abuse report ##{abuseId}] on #{instanceName} has been #{isAccepted ? 'accepted' : 'rejected'} diff --git a/server/lib/emails/account-abuse-new/html.pug b/server/lib/emails/account-abuse-new/html.pug deleted file mode 100644 index e4c0366fb..000000000 --- a/server/lib/emails/account-abuse-new/html.pug +++ /dev/null @@ -1,14 +0,0 @@ -extends ../common/greetings -include ../common/mixins.pug - -block title - | An account is pending moderation - -block content - p - | #[a(href=WEBSERVER.URL) #{instanceName}] received an abuse report for the #{isLocal ? '' : 'remote '}account - a(href=accountUrl) #{accountDisplayName} - - p The reporter, #{reporter}, cited the following reason(s): - blockquote #{reason} - br(style="display: none;") diff --git a/server/lib/emails/common/base.pug b/server/lib/emails/common/base.pug deleted file mode 100644 index 41e94564d..000000000 --- a/server/lib/emails/common/base.pug +++ /dev/null @@ -1,258 +0,0 @@ -//- - The email background color is defined in three places: - 1. body tag: for most email clients - 2. center tag: for Gmail and Inbox mobile apps and web versions of Gmail, GSuite, Inbox, Yahoo, AOL, Libero, Comcast, freenet, Mail.ru, Orange.fr - 3. mso conditional: For Windows 10 Mail -- var backgroundColor = "#fff"; -- var mainColor = "#f2690d"; -doctype html -head - // This template is heavily adapted from the Cerberus Fluid template. Kudos to them! - meta(charset='utf-8') - //- utf-8 works for most cases - meta(name='viewport' content='width=device-width') - //- Forcing initial-scale shouldn't be necessary - meta(http-equiv='X-UA-Compatible' content='IE=edge') - //- Use the latest (edge) version of IE rendering engine - meta(name='x-apple-disable-message-reformatting') - //- Disable auto-scale in iOS 10 Mail entirely - meta(name='format-detection' content='telephone=no,address=no,email=no,date=no,url=no') - //- Tell iOS not to automatically link certain text strings. - meta(name='color-scheme' content='light') - meta(name='supported-color-schemes' content='light') - //- The title tag shows in email notifications, like Android 4.4. - title #{subject} - //- What it does: Makes background images in 72ppi Outlook render at correct size. - //if gte mso 9 - xml - o:officedocumentsettings - o:allowpng - o:pixelsperinch 96 - //- CSS Reset : BEGIN - style. - /* What it does: Tells the email client that only light styles are provided but the client can transform them to dark. A duplicate of meta color-scheme meta tag above. */ - :root { - color-scheme: light; - supported-color-schemes: light; - } - /* What it does: Remove spaces around the email design added by some email clients. */ - /* Beware: It can remove the padding / margin and add a background color to the compose a reply window. */ - html, - body { - margin: 0 auto !important; - padding: 0 !important; - height: 100% !important; - width: 100% !important; - } - /* What it does: Stops email clients resizing small text. */ - * { - -ms-text-size-adjust: 100%; - -webkit-text-size-adjust: 100%; - } - /* What it does: Centers email on Android 4.4 */ - div[style*="margin: 16px 0"] { - margin: 0 !important; - } - /* What it does: forces Samsung Android mail clients to use the entire viewport */ - #MessageViewBody, #MessageWebViewDiv{ - width: 100% !important; - } - /* What it does: Stops Outlook from adding extra spacing to tables. */ - table, - td { - mso-table-lspace: 0pt !important; - mso-table-rspace: 0pt !important; - } - /* What it does: Fixes webkit padding issue. */ - table { - border-spacing: 0 !important; - border-collapse: collapse !important; - table-layout: fixed !important; - margin: 0 auto !important; - } - /* What it does: Uses a better rendering method when resizing images in IE. */ - img { - -ms-interpolation-mode:bicubic; - } - /* What it does: Prevents Windows 10 Mail from underlining links despite inline CSS. Styles for underlined links should be inline. */ - a { - text-decoration: none; - } - a:not(.nocolor) { - color: #{mainColor}; - } - a.nocolor { - color: inherit !important; - } - /* What it does: A work-around for email clients meddling in triggered links. */ - a[x-apple-data-detectors], /* iOS */ - .unstyle-auto-detected-links a, - .aBn { - border-bottom: 0 !important; - cursor: default !important; - color: inherit !important; - text-decoration: none !important; - font-size: inherit !important; - font-family: inherit !important; - font-weight: inherit !important; - line-height: inherit !important; - } - /* What it does: Prevents Gmail from displaying a download button on large, non-linked images. */ - .a6S { - display: none !important; - opacity: 0.01 !important; - } - /* What it does: Prevents Gmail from changing the text color in conversation threads. */ - .im { - color: inherit !important; - } - /* If the above doesn't work, add a .g-img class to any image in question. */ - img.g-img + div { - display: none !important; - } - /* What it does: Removes right gutter in Gmail iOS app: https://github.com/TedGoas/Cerberus/issues/89 */ - /* Create one of these media queries for each additional viewport size you'd like to fix */ - /* iPhone 4, 4S, 5, 5S, 5C, and 5SE */ - @media only screen and (min-device-width: 320px) and (max-device-width: 374px) { - u ~ div .email-container { - min-width: 320px !important; - } - } - /* iPhone 6, 6S, 7, 8, and X */ - @media only screen and (min-device-width: 375px) and (max-device-width: 413px) { - u ~ div .email-container { - min-width: 375px !important; - } - } - /* iPhone 6+, 7+, and 8+ */ - @media only screen and (min-device-width: 414px) { - u ~ div .email-container { - min-width: 414px !important; - } - } - //- CSS Reset : END - //- CSS for PeerTube : START - style. - blockquote { - margin-left: 0; - padding-left: 20px; - border-left: 2px solid #f2690d; - } - //- CSS for PeerTube : END - //- Progressive Enhancements : BEGIN - style. - /* What it does: Hover styles for buttons */ - .button-td, - .button-a { - transition: all 100ms ease-in; - } - .button-td-primary:hover, - .button-a-primary:hover { - background: #555555 !important; - border-color: #555555 !important; - } - /* Media Queries */ - @media screen and (max-width: 600px) { - /* What it does: Adjust typography on small screens to improve readability */ - .email-container p { - font-size: 17px !important; - } - } - //- Progressive Enhancements : END - -body(width="100%" style="margin: 0; padding: 0 !important; mso-line-height-rule: exactly; background-color: #{backgroundColor};") - center(role='article' aria-roledescription='email' lang='en' style='width: 100%; background-color: #{backgroundColor};') - //if mso | IE - table(role='presentation' border='0' cellpadding='0' cellspacing='0' width='100%' style='background-color: #fff;') - tr - td - //- Visually Hidden Preheader Text : BEGIN - div(style='max-height:0; overflow:hidden; mso-hide:all;' aria-hidden='true') - block preheader - //- Visually Hidden Preheader Text : END - - //- Create white space after the desired preview text so email clients don’t pull other distracting text into the inbox preview. Extend as necessary. - //- Preview Text Spacing Hack : BEGIN - div(style='display: none; font-size: 1px; line-height: 1px; max-height: 0px; max-width: 0px; opacity: 0; overflow: hidden; mso-hide: all; font-family: sans-serif;') - | ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ - //- Preview Text Spacing Hack : END - - //- - Set the email width. Defined in two places: - 1. max-width for all clients except Desktop Windows Outlook, allowing the email to squish on narrow but never go wider than 600px. - 2. MSO tags for Desktop Windows Outlook enforce a 600px width. - .email-container(style='max-width: 600px; margin: 0 auto;') - //if mso - table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' width='600') - tr - td - //- Email Body : BEGIN - table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%' style='margin: auto;') - //- 1 Column Text + Button : BEGIN - tr - td(style='background-color: #ffffff;') - table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%') - tr - td(style='padding: 20px; font-family: sans-serif; font-size: 15px; line-height: 20px; color: #555555;') - table(role="presentation" border="0" cellpadding="0" cellspacing="0" width="100%") - tr - td(width="40px") - img(src=`${WEBSERVER.URL}/client/assets/images/icons/icon-192x192.png` width="auto" height="30px" alt="" border="0" style="height: 30px; background: #ffffff; font-family: sans-serif; font-size: 15px; line-height: 15px; color: #555555;") - td - h1(style='margin: 10px 0 10px 0; font-family: sans-serif; font-size: 25px; line-height: 30px; color: #333333; font-weight: normal;') - block title - if title - | #{title} - else - | Something requires your attention - p(style='margin: 0;') - block body - if action - tr - td(style='padding: 0 20px;') - //- Button : BEGIN - table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' style='margin: auto;') - tr - td.button-td.button-td-primary(style='border-radius: 4px; background: #222222;') - a.button-a.button-a-primary(href=action.url style='background: #222222; border: 1px solid #000000; font-family: sans-serif; font-size: 15px; line-height: 15px; text-decoration: none; padding: 13px 17px; color: #ffffff; display: block; border-radius: 4px;') #{action.text} - //- Button : END - //- 1 Column Text + Button : END - //- Clear Spacer : BEGIN - tr - td(aria-hidden='true' height='20' style='font-size: 0px; line-height: 0px;') - br - //- Clear Spacer : END - //- Email Body : END - //- Email Footer : BEGIN - unless hideNotificationPreferencesLink - table(align='center' role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%' style='margin: auto;') - tr - td(style='padding: 20px; padding-bottom: 0px; font-family: sans-serif; font-size: 12px; line-height: 15px; text-align: center; color: #888888;') - webversion - a.nocolor(href=`${WEBSERVER.URL}/my-account/notifications` style='color: #cccccc; font-weight: bold;') View in your notifications - br - tr - td(style='padding: 20px; padding-top: 10px; font-family: sans-serif; font-size: 12px; line-height: 15px; text-align: center; color: #888888;') - unsubscribe - a.nocolor(href=`${WEBSERVER.URL}/my-account/settings#notifications` style='color: #888888;') Manage your notification preferences in your profile - br - //- Email Footer : END - //if mso - //- Full Bleed Background Section : BEGIN - table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%' style=`background-color: ${mainColor};`) - tr - td - .email-container(align='center' style='max-width: 600px; margin: auto;') - //if mso - table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='600' align='center') - tr - td - table(role='presentation' cellspacing='0' cellpadding='0' border='0' width='100%') - tr - td(style='padding: 20px; text-align: left; font-family: sans-serif; font-size: 12px; line-height: 20px; color: #ffffff;') - table(role="presentation" border="0" cellpadding="0" cellspacing="0" width="100%") - tr - td(valign="top") #[a(href="https://github.com/Chocobozzz/PeerTube" style="color: white !important") PeerTube © 2015-#{new Date().getFullYear()}] #[a(href="https://github.com/Chocobozzz/PeerTube/blob/master/CREDITS.md" style="color: white !important") PeerTube Contributors] - //if mso - //- Full Bleed Background Section : END - //if mso | IE diff --git a/server/lib/emails/common/greetings.pug b/server/lib/emails/common/greetings.pug deleted file mode 100644 index 5efe29dfb..000000000 --- a/server/lib/emails/common/greetings.pug +++ /dev/null @@ -1,11 +0,0 @@ -extends base - -block body - if username - p Hi #{username}, - else - p Hi, - block content - p - | Cheers,#[br] - | #{EMAIL.BODY.SIGNATURE} \ No newline at end of file diff --git a/server/lib/emails/common/html.pug b/server/lib/emails/common/html.pug deleted file mode 100644 index d76168b85..000000000 --- a/server/lib/emails/common/html.pug +++ /dev/null @@ -1,4 +0,0 @@ -extends greetings - -block content - p !{text} \ No newline at end of file diff --git a/server/lib/emails/common/mixins.pug b/server/lib/emails/common/mixins.pug deleted file mode 100644 index 831211864..000000000 --- a/server/lib/emails/common/mixins.pug +++ /dev/null @@ -1,7 +0,0 @@ -mixin channel(channel) - - var handle = `${channel.name}@${channel.host}` - | #[a(href=`${WEBSERVER.URL}/video-channels/${handle}` title=handle) #{channel.displayName}] - -mixin account(account) - - var handle = `${account.name}@${account.host}` - | #[a(href=`${WEBSERVER.URL}/accounts/${handle}` title=handle) #{account.displayName}] diff --git a/server/lib/emails/contact-form/html.pug b/server/lib/emails/contact-form/html.pug deleted file mode 100644 index 5a24fa6f1..000000000 --- a/server/lib/emails/contact-form/html.pug +++ /dev/null @@ -1,9 +0,0 @@ -extends ../common/greetings - -block title - | Someone just used the contact form - -block content - p #{fromName} sent you a message via the contact form on #[a(href=WEBSERVER.URL) #{instanceName}]: - blockquote(style='white-space: pre-wrap') #{body} - p You can contact them at #[a(href=`mailto:${fromEmail}`) #{fromEmail}], or simply reply to this email to get in touch. \ No newline at end of file diff --git a/server/lib/emails/follower-on-channel/html.pug b/server/lib/emails/follower-on-channel/html.pug deleted file mode 100644 index 8a352e90f..000000000 --- a/server/lib/emails/follower-on-channel/html.pug +++ /dev/null @@ -1,9 +0,0 @@ -extends ../common/greetings - -block title - | New follower on your channel - -block content - p. - Your #{followType} #[a(href=followingUrl) #{followingName}] has a new subscriber: - #[a(href=followerUrl) #{followerName}]. \ No newline at end of file diff --git a/server/lib/emails/password-create/html.pug b/server/lib/emails/password-create/html.pug deleted file mode 100644 index afa30ae97..000000000 --- a/server/lib/emails/password-create/html.pug +++ /dev/null @@ -1,10 +0,0 @@ -extends ../common/greetings - -block title - | Password creation for your account - -block content - p. - Welcome to #[a(href=WEBSERVER.URL) #{instanceName}]. Your username is: #{username}. - Please set your password by following #[a(href=createPasswordUrl) this link]: #[a(href=createPasswordUrl) #{createPasswordUrl}] - (this link will expire within seven days). \ No newline at end of file diff --git a/server/lib/emails/password-reset/html.pug b/server/lib/emails/password-reset/html.pug deleted file mode 100644 index 2af2885bc..000000000 --- a/server/lib/emails/password-reset/html.pug +++ /dev/null @@ -1,12 +0,0 @@ -extends ../common/greetings - -block title - | Password reset for your account - -block content - p. - A reset password procedure for your account #{username} has been requested on #[a(href=WEBSERVER.URL) #{instanceName}]. - Please follow #[a(href=resetPasswordUrl) this link] to reset it: #[a(href=resetPasswordUrl) #{resetPasswordUrl}] - (the link will expire within 1 hour). - p. - If you are not the person who initiated this request, please ignore this email. diff --git a/server/lib/emails/peertube-version-new/html.pug b/server/lib/emails/peertube-version-new/html.pug deleted file mode 100644 index 2f4d9399d..000000000 --- a/server/lib/emails/peertube-version-new/html.pug +++ /dev/null @@ -1,9 +0,0 @@ -extends ../common/greetings - -block title - | New PeerTube version available - -block content - p - | A new version of PeerTube is available: #{latestVersion}. - | You can check the latest news on #[a(href="https://joinpeertube.org/news") JoinPeerTube]. diff --git a/server/lib/emails/plugin-version-new/html.pug b/server/lib/emails/plugin-version-new/html.pug deleted file mode 100644 index 86d3d87e8..000000000 --- a/server/lib/emails/plugin-version-new/html.pug +++ /dev/null @@ -1,9 +0,0 @@ -extends ../common/greetings - -block title - | New plugin version available - -block content - p - | A new version of the plugin/theme #{pluginName} is available: #{latestVersion}. - | You might want to upgrade it on #[a(href=pluginUrl) the PeerTube admin interface]. diff --git a/server/lib/emails/user-registered/html.pug b/server/lib/emails/user-registered/html.pug deleted file mode 100644 index 20f62125e..000000000 --- a/server/lib/emails/user-registered/html.pug +++ /dev/null @@ -1,10 +0,0 @@ -extends ../common/greetings - -block title - | A new user registered - -block content - - var mail = user.email || user.pendingEmail; - p - | User #[a(href=`${WEBSERVER.URL}/accounts/${user.username}`) #{user.username}] just registered. - | You might want to contact them at #[a(href=`mailto:${mail}`) #{mail}]. \ No newline at end of file diff --git a/server/lib/emails/user-registration-request-accepted/html.pug b/server/lib/emails/user-registration-request-accepted/html.pug deleted file mode 100644 index 7a52c3fe1..000000000 --- a/server/lib/emails/user-registration-request-accepted/html.pug +++ /dev/null @@ -1,10 +0,0 @@ -extends ../common/greetings - -block title - | Congratulation #{username}, your registration request has been accepted! - -block content - p Your registration request has been accepted. - p Moderators sent you the following message: - blockquote(style='white-space: pre-wrap') #{moderationResponse} - p Your account has been created and you can login on #[a(href=loginLink) #{loginLink}] diff --git a/server/lib/emails/user-registration-request-rejected/html.pug b/server/lib/emails/user-registration-request-rejected/html.pug deleted file mode 100644 index ec0aa8dfe..000000000 --- a/server/lib/emails/user-registration-request-rejected/html.pug +++ /dev/null @@ -1,9 +0,0 @@ -extends ../common/greetings - -block title - | Registration request of your account #{username} has rejected - -block content - p Your registration request has been rejected. - p Moderators sent you the following message: - blockquote(style='white-space: pre-wrap') #{moderationResponse} diff --git a/server/lib/emails/user-registration-request/html.pug b/server/lib/emails/user-registration-request/html.pug deleted file mode 100644 index 64898f3f2..000000000 --- a/server/lib/emails/user-registration-request/html.pug +++ /dev/null @@ -1,9 +0,0 @@ -extends ../common/greetings - -block title - | A new user wants to register - -block content - p User #{registration.username} wants to register on your PeerTube instance with the following reason: - blockquote(style='white-space: pre-wrap') #{registration.registrationReason} - p You can accept or reject the registration request in the #[a(href=`${WEBSERVER.URL}/admin/moderation/registrations/list`) administration]. diff --git a/server/lib/emails/verify-email/html.pug b/server/lib/emails/verify-email/html.pug deleted file mode 100644 index 19ef65f75..000000000 --- a/server/lib/emails/verify-email/html.pug +++ /dev/null @@ -1,19 +0,0 @@ -extends ../common/greetings - -block title - | Email verification - -block content - if isRegistrationRequest - p You just requested an account on #[a(href=WEBSERVER.URL) #{instanceName}]. - else - p You just created an account on #[a(href=WEBSERVER.URL) #{instanceName}]. - - if isRegistrationRequest - p To complete your registration request you must verify your email first! - else - p To start using your account you must verify your email first! - - p Please follow #[a(href=verifyEmailUrl) this link] to verify this email belongs to you. - p If you can't see the verification link above you can use the following link #[a(href=verifyEmailUrl) #{verifyEmailUrl}] - p If you are not the person who initiated this request, please ignore this email. diff --git a/server/lib/emails/video-abuse-new/html.pug b/server/lib/emails/video-abuse-new/html.pug deleted file mode 100644 index a085b4b38..000000000 --- a/server/lib/emails/video-abuse-new/html.pug +++ /dev/null @@ -1,18 +0,0 @@ -extends ../common/greetings -include ../common/mixins.pug - -block title - | A video is pending moderation - -block content - p - | #[a(href=WEBSERVER.URL) #{instanceName}] received an abuse report for the #{isLocal ? '' : 'remote '}video " - a(href=videoUrl) #{videoName} - | " by #[+channel(videoChannel)] - if videoPublishedAt - | , published the #{videoPublishedAt}. - else - | , uploaded the #{videoCreatedAt} but not yet published. - p The reporter, #{reporter}, cited the following reason(s): - blockquote #{reason} - br(style="display: none;") diff --git a/server/lib/emails/video-auto-blacklist-new/html.pug b/server/lib/emails/video-auto-blacklist-new/html.pug deleted file mode 100644 index 07c8dfd16..000000000 --- a/server/lib/emails/video-auto-blacklist-new/html.pug +++ /dev/null @@ -1,17 +0,0 @@ -extends ../common/greetings -include ../common/mixins - -block title - | A video is pending moderation - -block content - p - | A recently added video was auto-blacklisted and requires moderator review before going public: - | - a(href=videoUrl) #{videoName} - | - | by #[+channel(channel)]. - p. - Apart from the publisher and the moderation team, no one will be able to see the video until you - unblacklist it. If you trust the publisher, any admin can whitelist the user for later videos so - that they don't require approval before going public. diff --git a/server/lib/emails/video-comment-abuse-new/html.pug b/server/lib/emails/video-comment-abuse-new/html.pug deleted file mode 100644 index 752bf7c10..000000000 --- a/server/lib/emails/video-comment-abuse-new/html.pug +++ /dev/null @@ -1,16 +0,0 @@ -extends ../common/greetings -include ../common/mixins.pug - -block title - | A comment is pending moderation - -block content - p - | #[a(href=WEBSERVER.URL) #{instanceName}] received an abuse report for the #{isLocal ? '' : 'remote '} - a(href=commentUrl) comment on video "#{videoName}" - | of #{flaggedAccount} - | created on #{commentCreatedAt} - - p The reporter, #{reporter}, cited the following reason(s): - blockquote #{reason} - br(style="display: none;") diff --git a/server/lib/emails/video-comment-mention/html.pug b/server/lib/emails/video-comment-mention/html.pug deleted file mode 100644 index a34c6b090..000000000 --- a/server/lib/emails/video-comment-mention/html.pug +++ /dev/null @@ -1,11 +0,0 @@ -extends ../common/greetings - -block title - | Someone mentioned you - -block content - p. - #[a(href=accountUrl title=handle) #{accountName}] mentioned you in a comment on video - "#[a(href=videoUrl) #{video.name}]": - blockquote !{commentHtml} - br(style="display: none;") diff --git a/server/lib/emails/video-comment-new/html.pug b/server/lib/emails/video-comment-new/html.pug deleted file mode 100644 index cbb683fee..000000000 --- a/server/lib/emails/video-comment-new/html.pug +++ /dev/null @@ -1,11 +0,0 @@ -extends ../common/greetings - -block title - | Someone commented your video - -block content - p. - #[a(href=accountUrl title=handle) #{accountName}] added a comment on your video - "#[a(href=videoUrl) #{video.name}]": - blockquote !{commentHtml} - br(style="display: none;") diff --git a/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/lib/files-cache/avatar-permanent-file-cache.ts deleted file mode 100644 index 0c508b063..000000000 --- a/server/lib/files-cache/avatar-permanent-file-cache.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { MActorImage } from '@server/types/models' -import { AbstractPermanentFileCache } from './shared' - -export class AvatarPermanentFileCache extends AbstractPermanentFileCache { - - constructor () { - super(CONFIG.STORAGE.ACTOR_IMAGES_DIR) - } - - protected loadModel (filename: string) { - return ActorImageModel.loadByName(filename) - } - - protected getImageSize (image: MActorImage): { width: number, height: number } { - if (image.width && image.height) { - return { - height: image.height, - width: image.width - } - } - - return ACTOR_IMAGES_SIZE[image.type][0] - } -} diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts deleted file mode 100644 index 5630a9b80..000000000 --- a/server/lib/files-cache/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './avatar-permanent-file-cache' -export * from './video-miniature-permanent-file-cache' -export * from './video-captions-simple-file-cache' -export * from './video-previews-simple-file-cache' -export * from './video-storyboards-simple-file-cache' -export * from './video-torrents-simple-file-cache' diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts deleted file mode 100644 index f990e9872..000000000 --- a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts +++ /dev/null @@ -1,132 +0,0 @@ -import express from 'express' -import { LRUCache } from 'lru-cache' -import { Model } from 'sequelize' -import { logger } from '@server/helpers/logger' -import { CachePromise } from '@server/helpers/promise-cache' -import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants' -import { downloadImageFromWorker } from '@server/lib/worker/parent-process' -import { HttpStatusCode } from '@shared/models' - -type ImageModel = { - fileUrl: string - filename: string - onDisk: boolean - - isOwned (): boolean - getPath (): string - - save (): Promise -} - -export abstract class AbstractPermanentFileCache { - // Unsafe because it can return paths that do not exist anymore - private readonly filenameToPathUnsafeCache = new LRUCache({ - max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE - }) - - protected abstract getImageSize (image: M): { width: number, height: number } - protected abstract loadModel (filename: string): Promise - - constructor (private readonly directory: string) { - - } - - async lazyServe (options: { - filename: string - res: express.Response - next: express.NextFunction - }) { - const { filename, res, next } = options - - if (this.filenameToPathUnsafeCache.has(filename)) { - return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) - } - - const image = await this.lazyLoadIfNeeded(filename) - if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - const path = image.getPath() - this.filenameToPathUnsafeCache.set(filename, path) - - return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => { - if (!err) return - - this.onServeError({ err, image, next, filename }) - }) - } - - @CachePromise({ - keyBuilder: filename => filename - }) - private async lazyLoadIfNeeded (filename: string) { - const image = await this.loadModel(filename) - if (!image) return undefined - - if (image.onDisk === false) { - if (!image.fileUrl) return undefined - - try { - await this.downloadRemoteFile(image) - } catch (err) { - logger.warn('Cannot process remote image %s.', image.fileUrl, { err }) - - return undefined - } - } - - return image - } - - async downloadRemoteFile (image: M) { - logger.info('Download remote image %s lazily.', image.fileUrl) - - const destination = await this.downloadImage({ - filename: image.filename, - fileUrl: image.fileUrl, - size: this.getImageSize(image) - }) - - image.onDisk = true - image.save() - .catch(err => logger.error('Cannot save new image disk state.', { err })) - - return destination - } - - private onServeError (options: { - err: any - image: M - filename: string - next: express.NextFunction - }) { - const { err, image, filename, next } = options - - // It seems this actor image is not on the disk anymore - if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) { - logger.error('Cannot lazy serve image %s.', filename, { err }) - - this.filenameToPathUnsafeCache.delete(filename) - - image.onDisk = false - image.save() - .catch(err => logger.error('Cannot save new image disk state.', { err })) - } - - return next(err) - } - - private downloadImage (options: { - fileUrl: string - filename: string - size: { width: number, height: number } - }) { - const downloaderOptions = { - url: options.fileUrl, - destDir: this.directory, - destName: options.filename, - size: options.size - } - - return downloadImageFromWorker(downloaderOptions) - } -} diff --git a/server/lib/files-cache/shared/abstract-simple-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts deleted file mode 100644 index 6fab322cd..000000000 --- a/server/lib/files-cache/shared/abstract-simple-file-cache.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { remove } from 'fs-extra' -import { logger } from '../../../helpers/logger' -import memoizee from 'memoizee' - -type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined - -export abstract class AbstractSimpleFileCache { - - getFilePath: (params: T) => Promise - - abstract getFilePathImpl (params: T): Promise - - // Load and save the remote file, then return the local path from filesystem - protected abstract loadRemoteFile (key: string): Promise - - init (max: number, maxAge: number) { - this.getFilePath = memoizee(this.getFilePathImpl, { - maxAge, - max, - promise: true, - dispose: (result?: GetFilePathResult) => { - if (result && result.isOwned !== true) { - remove(result.path) - .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name)) - .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err })) - } - } - }) - } -} diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts deleted file mode 100644 index 61c4aacc7..000000000 --- a/server/lib/files-cache/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './abstract-permanent-file-cache' -export * from './abstract-simple-file-cache' diff --git a/server/lib/files-cache/video-captions-simple-file-cache.ts b/server/lib/files-cache/video-captions-simple-file-cache.ts deleted file mode 100644 index cbeeff732..000000000 --- a/server/lib/files-cache/video-captions-simple-file-cache.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { CONFIG } from '../../initializers/config' -import { FILES_CACHE } from '../../initializers/constants' -import { VideoModel } from '../../models/video/video' -import { VideoCaptionModel } from '../../models/video/video-caption' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' - -class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoCaptionsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename) - if (!videoCaption) return undefined - - if (videoCaption.isOwned()) { - return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } - } - - return this.loadRemoteFile(filename) - } - - // Key is the caption filename - protected async loadRemoteFile (key: string) { - const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(key) - if (!videoCaption) return undefined - - if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') - - // Used to fetch the path - const video = await VideoModel.loadFull(videoCaption.videoId) - if (!video) return undefined - - const remoteUrl = videoCaption.getFileUrl(video) - const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - return { isOwned: false, path: destPath } - } catch (err) { - logger.info('Cannot fetch remote caption file %s.', remoteUrl, { err }) - - return undefined - } - } -} - -export { - VideoCaptionsSimpleFileCache -} diff --git a/server/lib/files-cache/video-miniature-permanent-file-cache.ts b/server/lib/files-cache/video-miniature-permanent-file-cache.ts deleted file mode 100644 index 35d9466f7..000000000 --- a/server/lib/files-cache/video-miniature-permanent-file-cache.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { THUMBNAILS_SIZE } from '@server/initializers/constants' -import { ThumbnailModel } from '@server/models/video/thumbnail' -import { MThumbnail } from '@server/types/models' -import { ThumbnailType } from '@shared/models' -import { AbstractPermanentFileCache } from './shared' - -export class VideoMiniaturePermanentFileCache extends AbstractPermanentFileCache { - - constructor () { - super(CONFIG.STORAGE.THUMBNAILS_DIR) - } - - protected loadModel (filename: string) { - return ThumbnailModel.loadByFilename(filename, ThumbnailType.MINIATURE) - } - - protected getImageSize (image: MThumbnail): { width: number, height: number } { - if (image.width && image.height) { - return { - height: image.height, - width: image.width - } - } - - return THUMBNAILS_SIZE - } -} diff --git a/server/lib/files-cache/video-previews-simple-file-cache.ts b/server/lib/files-cache/video-previews-simple-file-cache.ts deleted file mode 100644 index a05e80e16..000000000 --- a/server/lib/files-cache/video-previews-simple-file-cache.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { join } from 'path' -import { FILES_CACHE } from '../../initializers/constants' -import { VideoModel } from '../../models/video/video' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { ThumbnailModel } from '@server/models/video/thumbnail' -import { ThumbnailType } from '@shared/models' -import { logger } from '@server/helpers/logger' - -class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoPreviewsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const thumbnail = await ThumbnailModel.loadWithVideoByFilename(filename, ThumbnailType.PREVIEW) - if (!thumbnail) return undefined - - if (thumbnail.Video.isOwned()) return { isOwned: true, path: thumbnail.getPath() } - - return this.loadRemoteFile(thumbnail.Video.uuid) - } - - // Key is the video UUID - protected async loadRemoteFile (key: string) { - const video = await VideoModel.loadFull(key) - if (!video) return undefined - - if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') - - const preview = video.getPreview() - const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename) - const remoteUrl = preview.getOriginFileUrl(video) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath) - - return { isOwned: false, path: destPath } - } catch (err) { - logger.info('Cannot fetch remote preview file %s.', remoteUrl, { err }) - - return undefined - } - } -} - -export { - VideoPreviewsSimpleFileCache -} diff --git a/server/lib/files-cache/video-storyboards-simple-file-cache.ts b/server/lib/files-cache/video-storyboards-simple-file-cache.ts deleted file mode 100644 index 4cd96e70c..000000000 --- a/server/lib/files-cache/video-storyboards-simple-file-cache.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { StoryboardModel } from '@server/models/video/storyboard' -import { FILES_CACHE } from '../../initializers/constants' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' - -class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoStoryboardsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const storyboard = await StoryboardModel.loadWithVideoByFilename(filename) - if (!storyboard) return undefined - - if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() } - - return this.loadRemoteFile(storyboard.filename) - } - - // Key is the storyboard filename - protected async loadRemoteFile (key: string) { - const storyboard = await StoryboardModel.loadWithVideoByFilename(key) - if (!storyboard) return undefined - - const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename) - const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath) - - return { isOwned: false, path: destPath } - } catch (err) { - logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err }) - - return undefined - } - } -} - -export { - VideoStoryboardsSimpleFileCache -} diff --git a/server/lib/files-cache/video-torrents-simple-file-cache.ts b/server/lib/files-cache/video-torrents-simple-file-cache.ts deleted file mode 100644 index 8bcd0b9bf..000000000 --- a/server/lib/files-cache/video-torrents-simple-file-cache.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { VideoFileModel } from '@server/models/video/video-file' -import { MVideo, MVideoFile } from '@server/types/models' -import { CONFIG } from '../../initializers/config' -import { FILES_CACHE } from '../../initializers/constants' -import { VideoModel } from '../../models/video/video' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' - -class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoTorrentsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(filename) - if (!file) return undefined - - if (file.getVideo().isOwned()) { - const downloadName = this.buildDownloadName(file.getVideo(), file) - - return { isOwned: true, path: join(CONFIG.STORAGE.TORRENTS_DIR, file.torrentFilename), downloadName } - } - - return this.loadRemoteFile(filename) - } - - // Key is the torrent filename - protected async loadRemoteFile (key: string) { - const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(key) - if (!file) return undefined - - if (file.getVideo().isOwned()) throw new Error('Cannot load remote file of owned video.') - - // Used to fetch the path - const video = await VideoModel.loadFull(file.getVideo().id) - if (!video) return undefined - - const remoteUrl = file.getRemoteTorrentUrl(video) - const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - const downloadName = this.buildDownloadName(video, file) - - return { isOwned: false, path: destPath, downloadName } - } catch (err) { - logger.info('Cannot fetch remote torrent file %s.', remoteUrl, { err }) - - return undefined - } - } - - private buildDownloadName (video: MVideo, file: MVideoFile) { - return `${video.name}-${file.resolution}p.torrent` - } -} - -export { - VideoTorrentsSimpleFileCache -} diff --git a/server/lib/hls.ts b/server/lib/hls.ts deleted file mode 100644 index 19044d7c2..000000000 --- a/server/lib/hls.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' -import { flatten } from 'lodash' -import PQueue from 'p-queue' -import { basename, dirname, join } from 'path' -import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' -import { uniqify, uuidRegex } from '@shared/core-utils' -import { sha256 } from '@shared/extra-utils' -import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' -import { VideoStorage } from '@shared/models' -import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' -import { logger, loggerTagsFactory } from '../helpers/logger' -import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' -import { generateRandomString } from '../helpers/utils' -import { CONFIG } from '../initializers/config' -import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' -import { sequelizeTypescript } from '../initializers/database' -import { VideoFileModel } from '../models/video/video-file' -import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { storeHLSFileFromFilename } from './object-storage' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' -import { VideoPathManager } from './video-path-manager' - -const lTags = loggerTagsFactory('hls') - -async function updateStreamingPlaylistsInfohashesIfNeeded () { - const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() - - // Use separate SQL queries, because we could have many videos to update - for (const playlist of playlistsToUpdate) { - await sequelizeTypescript.transaction(async t => { - const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) - - playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) - playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - - await playlist.save({ transaction: t }) - }) - } -} - -async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { - try { - let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) - playlistWithFiles = await updateSha256VODSegments(video, playlist) - - // Refresh playlist, operations can take some time - playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) - playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) - await playlistWithFiles.save() - - video.setHLSPlaylist(playlistWithFiles) - } catch (err) { - logger.warn('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) - } -} - -// --------------------------------------------------------------------------- - -// Avoid concurrency issues when updating streaming playlist files -const playlistFilesQueue = new PQueue({ concurrency: 1 }) - -function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise { - return playlistFilesQueue.add(async () => { - const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - - const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - - for (const file of playlist.VideoFiles) { - const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - - await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { - const size = await getVideoStreamDimensionsInfo(videoFilePath) - - const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) - const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` - - let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` - if (file.fps) line += ',FRAME-RATE=' + file.fps - - const codecs = await Promise.all([ - getVideoStreamCodec(videoFilePath), - getAudioStreamCodec(videoFilePath) - ]) - - line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` - - masterPlaylists.push(line) - masterPlaylists.push(playlistFilename) - }) - } - - if (playlist.playlistFilename) { - await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) - } - playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) - - const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) - await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') - - logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid)) - - if (playlist.storage === VideoStorage.OBJECT_STORAGE) { - playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) - await remove(masterPlaylistPath) - } - - return playlist.save() - }) -} - -// --------------------------------------------------------------------------- - -function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise { - return playlistFilesQueue.add(async () => { - const json: { [filename: string]: { [range: string]: string } } = {} - - const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - - // For all the resolutions available for this video - for (const file of playlist.VideoFiles) { - const rangeHashes: { [range: string]: string } = {} - const fileWithPlaylist = file.withVideoOrPlaylist(playlist) - - await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { - - return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { - const playlistContent = await readFile(resolutionPlaylistPath) - const ranges = getRangesFromPlaylist(playlistContent.toString()) - - const fd = await open(videoPath, 'r') - for (const range of ranges) { - const buf = Buffer.alloc(range.length) - await read(fd, buf, 0, range.length, range.offset) - - rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) - } - await close(fd) - - const videoFilename = file.filename - json[videoFilename] = rangeHashes - }) - }) - } - - if (playlist.segmentsSha256Filename) { - await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) - } - playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) - - const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) - await outputJSON(outputPath, json) - - if (playlist.storage === VideoStorage.OBJECT_STORAGE) { - playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) - await remove(outputPath) - } - - return playlist.save() - }) -} - -// --------------------------------------------------------------------------- - -async function buildSha256Segment (segmentPath: string) { - const buf = await readFile(segmentPath) - return sha256(buf) -} - -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { - let timer - let remainingBodyKBLimit = bodyKBLimit - - logger.info('Importing HLS playlist %s', playlistUrl) - - return new Promise(async (res, rej) => { - const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) - - await ensureDir(tmpDirectory) - - timer = setTimeout(() => { - deleteTmpDirectory(tmpDirectory) - - return rej(new Error('HLS download timeout.')) - }, timeout) - - try { - // Fetch master playlist - const subPlaylistUrls = await fetchUniqUrls(playlistUrl) - - const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) - const fileUrls = uniqify(flatten(await Promise.all(subRequests))) - - logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) - - for (const fileUrl of fileUrls) { - const destPath = join(tmpDirectory, basename(fileUrl)) - - await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) - - const { size } = await stat(destPath) - remainingBodyKBLimit -= (size / 1000) - - logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) - } - - clearTimeout(timer) - - await move(tmpDirectory, destinationDir, { overwrite: true }) - - return res() - } catch (err) { - deleteTmpDirectory(tmpDirectory) - - return rej(err) - } - }) - - function deleteTmpDirectory (directory: string) { - remove(directory) - .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) - } - - async function fetchUniqUrls (playlistUrl: string) { - const { body } = await doRequest(playlistUrl) - - if (!body) return [] - - const urls = body.split('\n') - .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) - .map(url => { - if (url.startsWith('http://') || url.startsWith('https://')) return url - - return `${dirname(playlistUrl)}/${url}` - }) - - return uniqify(urls) - } -} - -// --------------------------------------------------------------------------- - -async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { - const content = await readFile(playlistPath, 'utf8') - - const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) - - await writeFile(playlistPath, newContent, 'utf8') -} - -// --------------------------------------------------------------------------- - -function injectQueryToPlaylistUrls (content: string, queryString: string) { - return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) -} - -// --------------------------------------------------------------------------- - -export { - updateMasterHLSPlaylist, - updateSha256VODSegments, - buildSha256Segment, - downloadPlaylistSegments, - updateStreamingPlaylistsInfohashesIfNeeded, - updatePlaylistAfterFileChange, - injectQueryToPlaylistUrls, - renameVideoFileInPlaylist -} - -// --------------------------------------------------------------------------- - -function getRangesFromPlaylist (playlistContent: string) { - const ranges: { offset: number, length: number }[] = [] - const lines = playlistContent.split('\n') - const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ - - for (const line of lines) { - const captured = regex.exec(line) - - if (captured) { - ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) - } - } - - return ranges -} diff --git a/server/lib/internal-event-emitter.ts b/server/lib/internal-event-emitter.ts deleted file mode 100644 index 08b46a5c3..000000000 --- a/server/lib/internal-event-emitter.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { MChannel, MVideo } from '@server/types/models' -import { EventEmitter } from 'events' - -export interface PeerTubeInternalEvents { - 'video-created': (options: { video: MVideo }) => void - 'video-updated': (options: { video: MVideo }) => void - 'video-deleted': (options: { video: MVideo }) => void - - 'channel-created': (options: { channel: MChannel }) => void - 'channel-updated': (options: { channel: MChannel }) => void - 'channel-deleted': (options: { channel: MChannel }) => void -} - -declare interface InternalEventEmitter { - on( - event: U, listener: PeerTubeInternalEvents[U] - ): this - - emit( - event: U, ...args: Parameters - ): boolean -} - -class InternalEventEmitter extends EventEmitter { - - private static instance: InternalEventEmitter - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -export { - InternalEventEmitter -} diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts deleted file mode 100644 index 6ee9e2429..000000000 --- a/server/lib/job-queue/handlers/activitypub-cleaner.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { map } from 'bluebird' -import { Job } from 'bullmq' -import { - isAnnounceActivityValid, - isDislikeActivityValid, - isLikeActivityValid -} from '@server/helpers/custom-validators/activitypub/activity' -import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { AP_CLEANER } from '@server/initializers/constants' -import { fetchAP } from '@server/lib/activitypub/activity' -import { checkUrlsSameHost } from '@server/lib/activitypub/url' -import { Redis } from '@server/lib/redis' -import { VideoModel } from '@server/models/video/video' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { VideoShareModel } from '@server/models/video/video-share' -import { HttpStatusCode } from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' - -const lTags = loggerTagsFactory('ap-cleaner') - -// Job to clean remote interactions off local videos - -async function processActivityPubCleaner (_job: Job) { - logger.info('Processing ActivityPub cleaner.', lTags()) - - { - const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() - const { bodyValidator, deleter, updater } = rateOptionsFactory() - - await map(rateUrls, async rateUrl => { - // TODO: remove when https://github.com/mastodon/mastodon/issues/13571 is fixed - if (rateUrl.includes('#')) return - - const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter }) - - if (result?.status === 'deleted') { - const { videoId, type } = result.data - - await VideoModel.syncLocalRates(videoId, type, undefined) - } - }, { concurrency: AP_CLEANER.CONCURRENCY }) - } - - { - const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos() - const { bodyValidator, deleter, updater } = shareOptionsFactory() - - await map(shareUrls, async shareUrl => { - await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter }) - }, { concurrency: AP_CLEANER.CONCURRENCY }) - } - - { - const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos() - const { bodyValidator, deleter, updater } = commentOptionsFactory() - - await map(commentUrls, async commentUrl => { - await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter }) - }, { concurrency: AP_CLEANER.CONCURRENCY }) - } -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubCleaner -} - -// --------------------------------------------------------------------------- - -async function updateObjectIfNeeded (options: { - url: string - bodyValidator: (body: any) => boolean - updater: (url: string, newUrl: string) => Promise - deleter: (url: string) => Promise } -): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { - const { url, bodyValidator, updater, deleter } = options - - const on404OrTombstone = async () => { - logger.info('Removing remote AP object %s.', url, lTags(url)) - const data = await deleter(url) - - return { status: 'deleted' as 'deleted', data } - } - - try { - const { body } = await fetchAP(url) - - // If not same id, check same host and update - if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) - - if (body.type === 'Tombstone') { - return on404OrTombstone() - } - - const newUrl = body.id - if (newUrl !== url) { - if (checkUrlsSameHost(newUrl, url) !== true) { - throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) - } - - logger.info('Updating remote AP object %s.', url, lTags(url)) - const data = await updater(url, newUrl) - - return { status: 'updated', data } - } - - return null - } catch (err) { - // Does not exist anymore, remove entry - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - return on404OrTombstone() - } - - logger.debug('Remote AP object %s is unavailable.', url, lTags(url)) - - const unavailability = await Redis.Instance.addAPUnavailability(url) - if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) { - logger.info('Removing unavailable AP resource %s.', url, lTags(url)) - return on404OrTombstone() - } - - return null - } -} - -function rateOptionsFactory () { - return { - bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body), - - updater: async (url: string, newUrl: string) => { - const rate = await AccountVideoRateModel.loadByUrl(url, undefined) - rate.url = newUrl - - const videoId = rate.videoId - const type = rate.type - - await rate.save() - - return { videoId, type } - }, - - deleter: async (url) => { - const rate = await AccountVideoRateModel.loadByUrl(url, undefined) - - const videoId = rate.videoId - const type = rate.type - - await rate.destroy() - - return { videoId, type } - } - } -} - -function shareOptionsFactory () { - return { - bodyValidator: (body: any) => isAnnounceActivityValid(body), - - updater: async (url: string, newUrl: string) => { - const share = await VideoShareModel.loadByUrl(url, undefined) - share.url = newUrl - - await share.save() - - return undefined - }, - - deleter: async (url) => { - const share = await VideoShareModel.loadByUrl(url, undefined) - - await share.destroy() - - return undefined - } - } -} - -function commentOptionsFactory () { - return { - bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body), - - updater: async (url: string, newUrl: string) => { - const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) - comment.url = newUrl - - await comment.save() - - return undefined - }, - - deleter: async (url) => { - const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) - - await comment.destroy() - - return undefined - } - } -} diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts deleted file mode 100644 index a68c32ba0..000000000 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { Job } from 'bullmq' -import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url' -import { ActivitypubFollowPayload } from '@shared/models' -import { sanitizeHost } from '../../../helpers/core-utils' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorModel } from '../../../models/actor/actor' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { MActor, MActorFull } from '../../../types/models' -import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors' -import { sendFollow } from '../../activitypub/send' -import { Notifier } from '../../notifier' - -async function processActivityPubFollow (job: Job) { - const payload = job.data as ActivitypubFollowPayload - const host = payload.host - - logger.info('Processing ActivityPub follow in job %s.', job.id) - - let targetActor: MActorFull - if (!host || host === WEBSERVER.HOST) { - targetActor = await ActorModel.loadLocalByName(payload.name) - } else { - const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) - const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) - targetActor = await getOrCreateAPActor(actorUrl, 'all') - } - - if (payload.assertIsChannel && !targetActor.VideoChannel) { - logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host) - return - } - - const fromActor = await ActorModel.load(payload.followerActorId) - - return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow) -} -// --------------------------------------------------------------------------- - -export { - processActivityPubFollow -} - -// --------------------------------------------------------------------------- - -async function follow (fromActor: MActor, targetActor: MActorFull, isAutoFollow = false) { - if (fromActor.id === targetActor.id) { - throw new Error('Follower is the same as target actor.') - } - - // Same server, direct accept - const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' - - const actorFollow = await sequelizeTypescript.transaction(async t => { - const [ actorFollow ] = await ActorFollowModel.findOrCreateCustom({ - byActor: fromActor, - state, - targetActor, - activityId: getLocalActorFollowActivityPubUrl(fromActor, targetActor), - transaction: t - }) - - // Send a notification to remote server if our follow is not already accepted - if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t) - - return actorFollow - }) - - const followerFull = await ActorModel.loadFull(fromActor.id) - - const actorFollowFull = Object.assign(actorFollow, { - ActorFollowing: targetActor, - ActorFollower: followerFull - }) - - if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) - if (isAutoFollow === true) Notifier.Instance.notifyOfAutoInstanceFollowing(actorFollowFull) - - return actorFollow -} diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts deleted file mode 100644 index 8904d086f..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Job } from 'bullmq' -import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send' -import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache' -import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process' -import { ActivitypubHttpBroadcastPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -// Prefer using a worker thread for HTTP requests because on high load we may have to sign many requests, which can be CPU intensive - -async function processActivityPubHttpSequentialBroadcast (job: Job) { - logger.info('Processing ActivityPub broadcast in job %s.', job.id) - - const requestOptions = await buildRequestOptions(job.data) - - const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) - - return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) -} - -async function processActivityPubParallelHttpBroadcast (job: Job) { - logger.info('Processing ActivityPub parallel broadcast in job %s.', job.id) - - const requestOptions = await buildRequestOptions(job.data) - - const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) - - return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubHttpSequentialBroadcast, - processActivityPubParallelHttpBroadcast -} - -// --------------------------------------------------------------------------- - -async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) { - const body = await computeBody(payload) - const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) - - return { - method: 'POST' as 'POST', - json: body, - httpSignature: httpSignatureOptions, - headers: buildGlobalHeaders(body) - } -} diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts deleted file mode 100644 index b6cb3c4a6..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Job } from 'bullmq' -import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoShareModel } from '../../../models/video/video-share' -import { MVideoFullLight } from '../../../types/models' -import { crawlCollectionPage } from '../../activitypub/crawl' -import { createAccountPlaylists } from '../../activitypub/playlists' -import { processActivities } from '../../activitypub/process' -import { addVideoShares } from '../../activitypub/share' -import { addVideoComments } from '../../activitypub/video-comments' - -async function processActivityPubHttpFetcher (job: Job) { - logger.info('Processing ActivityPub fetcher in job %s.', job.id) - - const payload = job.data as ActivitypubHttpFetcherPayload - - let video: MVideoFullLight - if (payload.videoId) video = await VideoModel.loadFull(payload.videoId) - - const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise } = { - 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }), - 'video-shares': items => addVideoShares(items, video), - 'video-comments': items => addVideoComments(items), - 'account-playlists': items => createAccountPlaylists(items) - } - - const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise } = { - 'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate), - 'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) - } - - return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type]) -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubHttpFetcher -} diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts deleted file mode 100644 index 50fca3f94..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Job } from 'bullmq' -import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send' -import { ActivitypubHttpUnicastPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { doRequest } from '../../../helpers/requests' -import { ActorFollowHealthCache } from '../../actor-follow-health-cache' - -async function processActivityPubHttpUnicast (job: Job) { - logger.info('Processing ActivityPub unicast in job %s.', job.id) - - const payload = job.data as ActivitypubHttpUnicastPayload - const uri = payload.uri - - const body = await computeBody(payload) - const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) - - const options = { - method: 'POST' as 'POST', - json: body, - httpSignature: httpSignatureOptions, - headers: buildGlobalHeaders(body) - } - - try { - await doRequest(uri, options) - ActorFollowHealthCache.Instance.updateActorFollowsHealth([ uri ], []) - } catch (err) { - ActorFollowHealthCache.Instance.updateActorFollowsHealth([], [ uri ]) - - throw err - } -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubHttpUnicast -} diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts deleted file mode 100644 index 706bf17fa..000000000 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Job } from 'bullmq' -import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists' -import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos' -import { loadVideoByUrl } from '@server/lib/model-loaders' -import { RefreshPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { ActorModel } from '../../../models/actor/actor' -import { VideoPlaylistModel } from '../../../models/video/video-playlist' -import { refreshActorIfNeeded } from '../../activitypub/actors' - -async function refreshAPObject (job: Job) { - const payload = job.data as RefreshPayload - - logger.info('Processing AP refresher in job %s for %s.', job.id, payload.url) - - if (payload.type === 'video') return refreshVideo(payload.url) - if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url) - if (payload.type === 'actor') return refreshActor(payload.url) -} - -// --------------------------------------------------------------------------- - -export { - refreshAPObject -} - -// --------------------------------------------------------------------------- - -async function refreshVideo (videoUrl: string) { - const fetchType = 'all' as 'all' - const syncParam = { rates: true, shares: true, comments: true } - - const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) - if (videoFromDatabase) { - const refreshOptions = { - video: videoFromDatabase, - fetchedType: fetchType, - syncParam - } - - await refreshVideoIfNeeded(refreshOptions) - } -} - -async function refreshActor (actorUrl: string) { - const fetchType = 'all' as 'all' - const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) - - if (actor) { - await refreshActorIfNeeded({ actor, fetchedType: fetchType }) - } -} - -async function refreshVideoPlaylist (playlistUrl: string) { - const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl) - - if (playlist) { - await refreshVideoPlaylistIfNeeded(playlist) - } -} diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts deleted file mode 100644 index 27a2d431b..000000000 --- a/server/lib/job-queue/handlers/actor-keys.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Job } from 'bullmq' -import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors' -import { ActorModel } from '@server/models/actor/actor' -import { ActorKeysPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processActorKeys (job: Job) { - const payload = job.data as ActorKeysPayload - logger.info('Processing actor keys in job %s.', job.id) - - const actor = await ActorModel.load(payload.actorId) - - await generateAndSaveActorKeys(actor) -} - -// --------------------------------------------------------------------------- - -export { - processActorKeys -} diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts deleted file mode 100644 index ffdd8c5b5..000000000 --- a/server/lib/job-queue/handlers/after-video-channel-import.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Job } from 'bullmq' -import { logger } from '@server/helpers/logger' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models' - -export async function processAfterVideoChannelImport (job: Job) { - const payload = job.data as AfterVideoChannelImportPayload - if (!payload.channelSyncId) return - - logger.info('Processing after video channel import in job %s.', job.id) - - const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId) - if (!sync) { - logger.error('Unknown sync id %d.', payload.channelSyncId) - return - } - - const childrenValues = await job.getChildrenValues() - - let errors = 0 - let successes = 0 - - for (const value of Object.values(childrenValues)) { - if (value.resultType === 'success') successes++ - else if (value.resultType === 'error') errors++ - } - - if (errors > 0) { - sync.state = VideoChannelSyncState.FAILED - logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes }) - } else { - sync.state = VideoChannelSyncState.SYNCED - logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes }) - } - - await sync.save() -} diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts deleted file mode 100644 index 567bcc076..000000000 --- a/server/lib/job-queue/handlers/email.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Job } from 'bullmq' -import { EmailPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { Emailer } from '../../emailer' - -async function processEmail (job: Job) { - const payload = job.data as EmailPayload - logger.info('Processing email in job %s.', job.id) - - return Emailer.Instance.sendMail(payload) -} - -// --------------------------------------------------------------------------- - -export { - processEmail -} diff --git a/server/lib/job-queue/handlers/federate-video.ts b/server/lib/job-queue/handlers/federate-video.ts deleted file mode 100644 index 6aac36741..000000000 --- a/server/lib/job-queue/handlers/federate-video.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Job } from 'bullmq' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { sequelizeTypescript } from '@server/initializers/database' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { VideoModel } from '@server/models/video/video' -import { FederateVideoPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -function processFederateVideo (job: Job) { - const payload = job.data as FederateVideoPayload - - logger.info('Processing video federation in job %s.', job.id) - - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(payload.videoUUID, t) - if (!video) return - - return federateVideoIfNeeded(video, payload.isNewVideo, t) - }) - }) -} - -// --------------------------------------------------------------------------- - -export { - processFederateVideo -} diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts deleted file mode 100644 index eea20274a..000000000 --- a/server/lib/job-queue/handlers/generate-storyboard.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { Job } from 'bullmq' -import { join } from 'path' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { generateImageFilename, getImageSize } from '@server/helpers/image-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { deleteFileAndCatch } from '@server/helpers/utils' -import { CONFIG } from '@server/initializers/config' -import { STORYBOARD } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { StoryboardModel } from '@server/models/video/storyboard' -import { VideoModel } from '@server/models/video/video' -import { MVideo } from '@server/types/models' -import { FFmpegImage, isAudioFile } from '@shared/ffmpeg' -import { GenerateStoryboardPayload } from '@shared/models' - -const lTagsBase = loggerTagsFactory('storyboard') - -async function processGenerateStoryboard (job: Job): Promise { - const payload = job.data as GenerateStoryboardPayload - const lTags = lTagsBase(payload.videoUUID) - - logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags) - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) - - try { - const video = await VideoModel.loadFull(payload.videoUUID) - if (!video) { - logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) - return - } - - const inputFile = video.getMaxQualityFile() - - await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { - const isAudio = await isAudioFile(videoPath) - - if (isAudio) { - logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) - return - } - - const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) - - const filename = generateImageFilename() - const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) - - const totalSprites = buildTotalSprites(video) - if (totalSprites === 0) { - logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags) - return - } - - const spriteDuration = Math.round(video.duration / totalSprites) - - const spritesCount = findGridSize({ - toFind: totalSprites, - maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT - }) - - logger.debug( - 'Generating storyboard from video of %s to %s', video.uuid, destination, - { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } - ) - - await ffmpeg.generateStoryboardFromVideo({ - destination, - path: videoPath, - sprites: { - size: STORYBOARD.SPRITE_SIZE, - count: spritesCount, - duration: spriteDuration - } - }) - - const imageSize = await getImageSize(destination) - - await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - const videoStillExists = await VideoModel.load(video.id, transaction) - if (!videoStillExists) { - logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) - deleteFileAndCatch(destination) - return - } - - const existing = await StoryboardModel.loadByVideo(video.id, transaction) - if (existing) await existing.destroy({ transaction }) - - await StoryboardModel.create({ - filename, - totalHeight: imageSize.height, - totalWidth: imageSize.width, - spriteHeight: STORYBOARD.SPRITE_SIZE.height, - spriteWidth: STORYBOARD.SPRITE_SIZE.width, - spriteDuration, - videoId: video.id - }, { transaction }) - - logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags) - - if (payload.federate) { - await federateVideoIfNeeded(video, false, transaction) - } - }) - }) - }) - } finally { - inputFileMutexReleaser() - } -} - -// --------------------------------------------------------------------------- - -export { - processGenerateStoryboard -} - -function buildTotalSprites (video: MVideo) { - const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width - const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) - - // We can generate a single line - if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites - - return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) -} - -function findGridSize (options: { - toFind: number - maxEdgeCount: number -}) { - const { toFind, maxEdgeCount } = options - - for (let i = 1; i <= maxEdgeCount; i++) { - for (let j = i; j <= maxEdgeCount; j++) { - if (toFind === i * j) return { width: j, height: i } - } - } - - throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`) -} - -function findGridFit (value: number, maxMultiplier: number) { - for (let i = value; i--; i > 0) { - if (!isPrimeWithin(i, maxMultiplier)) return i - } - - throw new Error('Could not find prime number below ' + value) -} - -function isPrimeWithin (value: number, maxMultiplier: number) { - if (value < 2) return false - - for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) { - if (value % i === 0 && value / i <= maxMultiplier) return false - } - - return true -} diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts deleted file mode 100644 index edf52de0c..000000000 --- a/server/lib/job-queue/handlers/manage-video-torrent.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { Job } from 'bullmq' -import { extractVideo } from '@server/helpers/video' -import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoModel } from '@server/models/video/video' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { ManageVideoTorrentPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processManageVideoTorrent (job: Job) { - const payload = job.data as ManageVideoTorrentPayload - logger.info('Processing torrent in job %s.', job.id) - - if (payload.action === 'create') return doCreateAction(payload) - if (payload.action === 'update-metadata') return doUpdateMetadataAction(payload) -} - -// --------------------------------------------------------------------------- - -export { - processManageVideoTorrent -} - -// --------------------------------------------------------------------------- - -async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'create' }) { - const [ video, file ] = await Promise.all([ - loadVideoOrLog(payload.videoId), - loadFileOrLog(payload.videoFileId) - ]) - - if (!video || !file) return - - const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await file.reload() - - await createTorrentAndSetInfoHash(video, file) - - // Refresh videoFile because the createTorrentAndSetInfoHash could be long - const refreshedFile = await VideoFileModel.loadWithVideo(file.id) - // File does not exist anymore, remove the generated torrent - if (!refreshedFile) return file.removeTorrent() - - refreshedFile.infoHash = file.infoHash - refreshedFile.torrentFilename = file.torrentFilename - - await refreshedFile.save() - } finally { - fileMutexReleaser() - } -} - -async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) { - const [ video, streamingPlaylist, file ] = await Promise.all([ - loadVideoOrLog(payload.videoId), - loadStreamingPlaylistOrLog(payload.streamingPlaylistId), - loadFileOrLog(payload.videoFileId) - ]) - - if ((!video && !streamingPlaylist) || !file) return - - const extractedVideo = extractVideo(video || streamingPlaylist) - const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid) - - try { - await updateTorrentMetadata(video || streamingPlaylist, file) - - await file.save() - } finally { - fileMutexReleaser() - } -} - -async function loadVideoOrLog (videoId: number) { - if (!videoId) return undefined - - const video = await VideoModel.load(videoId) - if (!video) { - logger.debug('Do not process torrent for video %d: does not exist anymore.', videoId) - } - - return video -} - -async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) { - if (!streamingPlaylistId) return undefined - - const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) - if (!streamingPlaylist) { - logger.debug('Do not process torrent for streaming playlist %d: does not exist anymore.', streamingPlaylistId) - } - - return streamingPlaylist -} - -async function loadFileOrLog (videoFileId: number) { - if (!videoFileId) return undefined - - const file = await VideoFileModel.load(videoFileId) - - if (!file) { - logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) - } - - return file -} diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts deleted file mode 100644 index 9a99b6722..000000000 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { Job } from 'bullmq' -import { remove } from 'fs-extra' -import { join } from 'path' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { updateTorrentMetadata } from '@server/helpers/webtorrent' -import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' -import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage' -import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' -import { MoveObjectStoragePayload, VideoState, VideoStorage } from '@shared/models' - -const lTagsBase = loggerTagsFactory('move-object-storage') - -export async function processMoveToObjectStorage (job: Job) { - const payload = job.data as MoveObjectStoragePayload - logger.info('Moving video %s in job %s.', payload.videoUUID, job.id) - - const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) - - const video = await VideoModel.loadWithFiles(payload.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID)) - fileMutexReleaser() - return undefined - } - - const lTags = lTagsBase(video.uuid, video.url) - - try { - if (video.VideoFiles) { - logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags) - - await moveWebVideoFiles(video) - } - - if (video.VideoStreamingPlaylists) { - logger.debug('Moving HLS playlist of %s.', video.uuid) - - await moveHLSFiles(video) - } - - const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') - if (pendingMove === 0) { - logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags) - - await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) - } - } catch (err) { - await onMoveToObjectStorageFailure(job, err) - - throw err - } finally { - fileMutexReleaser() - } - - return payload.videoUUID -} - -export async function onMoveToObjectStorageFailure (job: Job, err: any) { - const payload = job.data as MoveObjectStoragePayload - - const video = await VideoModel.loadWithFiles(payload.videoUUID) - if (!video) return - - logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) }) - - await moveToFailedMoveToObjectStorageState(video) - await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') -} - -// --------------------------------------------------------------------------- - -async function moveWebVideoFiles (video: MVideoWithAllFiles) { - for (const file of video.VideoFiles) { - if (file.storage !== VideoStorage.FILE_SYSTEM) continue - - const fileUrl = await storeWebVideoFile(video, file) - - const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) - await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) - } -} - -async function moveHLSFiles (video: MVideoWithAllFiles) { - for (const playlist of video.VideoStreamingPlaylists) { - const playlistWithVideo = playlist.withVideo(video) - - for (const file of playlist.VideoFiles) { - if (file.storage !== VideoStorage.FILE_SYSTEM) continue - - // Resolution playlist - const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - await storeHLSFileFromFilename(playlistWithVideo, playlistFilename) - - // Resolution fragmented file - const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename) - - const oldPath = join(getHLSDirectory(video), file.filename) - - await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) - } - } -} - -async function doAfterLastJob (options: { - video: MVideoWithAllFiles - previousVideoState: VideoState - isNewVideo: boolean -}) { - const { video, previousVideoState, isNewVideo } = options - - for (const playlist of video.VideoStreamingPlaylists) { - if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue - - const playlistWithVideo = playlist.withVideo(video) - - // Master playlist - playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) - // Sha256 segments file - playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) - - playlist.storage = VideoStorage.OBJECT_STORAGE - - playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles) - playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - - await playlist.save() - } - - // Remove empty hls video directory - if (video.VideoStreamingPlaylists) { - await remove(getHLSDirectory(video)) - } - - await moveToNextState({ video, previousVideoState, isNewVideo }) -} - -async function onFileMoved (options: { - videoOrPlaylist: MVideo | MStreamingPlaylistVideo - file: MVideoFile - fileUrl: string - oldPath: string -}) { - const { videoOrPlaylist, file, fileUrl, oldPath } = options - - file.fileUrl = fileUrl - file.storage = VideoStorage.OBJECT_STORAGE - - await updateTorrentMetadata(videoOrPlaylist, file) - await file.save() - - logger.debug('Removing %s because it\'s now on object storage', oldPath) - await remove(oldPath) -} diff --git a/server/lib/job-queue/handlers/notify.ts b/server/lib/job-queue/handlers/notify.ts deleted file mode 100644 index 83605396c..000000000 --- a/server/lib/job-queue/handlers/notify.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Job } from 'bullmq' -import { Notifier } from '@server/lib/notifier' -import { VideoModel } from '@server/models/video/video' -import { NotifyPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processNotify (job: Job) { - const payload = job.data as NotifyPayload - logger.info('Processing %s notification in job %s.', payload.action, job.id) - - if (payload.action === 'new-video') return doNotifyNewVideo(payload) -} - -// --------------------------------------------------------------------------- - -export { - processNotify -} - -// --------------------------------------------------------------------------- - -async function doNotifyNewVideo (payload: NotifyPayload & { action: 'new-video' }) { - const refreshedVideo = await VideoModel.loadFull(payload.videoUUID) - if (!refreshedVideo) return - - Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo) -} diff --git a/server/lib/job-queue/handlers/transcoding-job-builder.ts b/server/lib/job-queue/handlers/transcoding-job-builder.ts deleted file mode 100644 index 8621b109f..000000000 --- a/server/lib/job-queue/handlers/transcoding-job-builder.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Job } from 'bullmq' -import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { pick } from '@shared/core-utils' -import { TranscodingJobBuilderPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { JobQueue } from '../job-queue' - -async function processTranscodingJobBuilder (job: Job) { - const payload = job.data as TranscodingJobBuilderPayload - - logger.info('Processing transcoding job builder in job %s.', job.id) - - if (payload.optimizeJob) { - const video = await VideoModel.loadFull(payload.videoUUID) - const user = await UserModel.loadByVideoId(video.id) - const videoFile = video.getMaxQualityFile() - - await createOptimizeOrMergeAudioJobs({ - ...pick(payload.optimizeJob, [ 'isNewVideo' ]), - - video, - videoFile, - user, - videoFileAlreadyLocked: false - }) - } - - for (const job of (payload.jobs || [])) { - await JobQueue.Instance.createJob(job) - - await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') - } - - for (const sequentialJobs of (payload.sequentialJobs || [])) { - await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs) - - await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.filter(s => !!s).length) - } -} - -// --------------------------------------------------------------------------- - -export { - processTranscodingJobBuilder -} diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts deleted file mode 100644 index 035f88e96..000000000 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Job } from 'bullmq' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { synchronizeChannel } from '@server/lib/sync-channel' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { MChannelSync } from '@server/types/models' -import { VideoChannelImportPayload } from '@shared/models' - -export async function processVideoChannelImport (job: Job) { - const payload = job.data as VideoChannelImportPayload - - logger.info('Processing video channel import in job %s.', job.id) - - // Channel import requires only http upload to be allowed - if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { - throw new Error('Cannot import channel as the HTTP upload is disabled') - } - - if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { - throw new Error('Cannot import channel as the synchronization is disabled') - } - - let channelSync: MChannelSync - if (payload.partOfChannelSyncId) { - channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId) - - if (!channelSync) { - throw new Error('Unlnown channel sync specified in videos channel import') - } - } - - const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) - - logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) - - await synchronizeChannel({ - channel: videoChannel, - externalChannelUrl: payload.externalChannelUrl, - channelSync, - videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT - }) -} diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts deleted file mode 100644 index d221e8968..000000000 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Job } from 'bullmq' -import { copy, stat } from 'fs-extra' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { CONFIG } from '@server/initializers/config' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { generateWebVideoFilename } from '@server/lib/paths' -import { buildMoveToObjectStorageJob } from '@server/lib/video' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoModel } from '@server/models/video/video' -import { VideoFileModel } from '@server/models/video/video-file' -import { MVideoFullLight } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' -import { VideoFileImportPayload, VideoStorage } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { JobQueue } from '../job-queue' - -async function processVideoFileImport (job: Job) { - const payload = job.data as VideoFileImportPayload - logger.info('Processing video file import in job %s.', job.id) - - const video = await VideoModel.loadFull(payload.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Do not process job %d, video does not exist.', job.id) - return undefined - } - - await updateVideoFile(video, payload.filePath) - - if (CONFIG.OBJECT_STORAGE.ENABLED) { - await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState: video.state })) - } else { - await federateVideoIfNeeded(video, false) - } - - return video -} - -// --------------------------------------------------------------------------- - -export { - processVideoFileImport -} - -// --------------------------------------------------------------------------- - -async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { - const { resolution } = await getVideoStreamDimensionsInfo(inputFilePath) - const { size } = await stat(inputFilePath) - const fps = await getVideoStreamFPS(inputFilePath) - - const fileExt = getLowercaseExtension(inputFilePath) - - const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === resolution) - - if (currentVideoFile) { - // Remove old file and old torrent - await video.removeWebVideoFile(currentVideoFile) - // Remove the old video file from the array - video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) - - await currentVideoFile.destroy() - } - - const newVideoFile = new VideoFileModel({ - resolution, - extname: fileExt, - filename: generateWebVideoFilename(resolution, fileExt), - storage: VideoStorage.FILE_SYSTEM, - size, - fps, - videoId: video.id - }) - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) - await copy(inputFilePath, outputPath) - - video.VideoFiles.push(newVideoFile) - await createTorrentAndSetInfoHash(video, newVideoFile) - - await newVideoFile.save() -} diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts deleted file mode 100644 index e5cd258d6..000000000 --- a/server/lib/job-queue/handlers/video-import.ts +++ /dev/null @@ -1,344 +0,0 @@ -import { Job } from 'bullmq' -import { move, remove, stat } from 'fs-extra' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' -import { CONFIG } from '@server/initializers/config' -import { isPostImportVideoAccepted } from '@server/lib/moderation' -import { generateWebVideoFilename } from '@server/lib/paths' -import { Hooks } from '@server/lib/plugins/hooks' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' -import { isAbleToUploadVideo } from '@server/lib/user' -import { buildMoveToObjectStorageJob } from '@server/lib/video' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { buildNextVideoState } from '@server/lib/video-state' -import { ThumbnailModel } from '@server/models/video/thumbnail' -import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' -import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' -import { getLowercaseExtension } from '@shared/core-utils' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' -import { - ThumbnailType, - VideoImportPayload, - VideoImportPreventExceptionResult, - VideoImportState, - VideoImportTorrentPayload, - VideoImportTorrentPayloadType, - VideoImportYoutubeDLPayload, - VideoImportYoutubeDLPayloadType, - VideoResolution, - VideoState -} from '@shared/models' -import { logger } from '../../../helpers/logger' -import { getSecureTorrentName } from '../../../helpers/utils' -import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' -import { JOB_TTL } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { VideoModel } from '../../../models/video/video' -import { VideoFileModel } from '../../../models/video/video-file' -import { VideoImportModel } from '../../../models/video/video-import' -import { federateVideoIfNeeded } from '../../activitypub/videos' -import { Notifier } from '../../notifier' -import { generateLocalVideoMiniature } from '../../thumbnail' -import { JobQueue } from '../job-queue' - -async function processVideoImport (job: Job): Promise { - const payload = job.data as VideoImportPayload - - const videoImport = await getVideoImportOrDie(payload) - if (videoImport.state === VideoImportState.CANCELLED) { - logger.info('Do not process import since it has been cancelled', { payload }) - return { resultType: 'success' } - } - - videoImport.state = VideoImportState.PROCESSING - await videoImport.save() - - try { - if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload) - if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload) - - return { resultType: 'success' } - } catch (err) { - if (!payload.preventException) throw err - - logger.warn('Catch error in video import to send value to parent job.', { payload, err }) - return { resultType: 'error' } - } -} - -// --------------------------------------------------------------------------- - -export { - processVideoImport -} - -// --------------------------------------------------------------------------- - -async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) { - logger.info('Processing torrent video import in job %s.', job.id) - - const options = { type: payload.type, videoImportId: payload.videoImportId } - - const target = { - torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, - uri: videoImport.magnetUri - } - return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options) -} - -async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { - logger.info('Processing youtubeDL video import in job %s.', job.id) - - const options = { type: payload.type, videoImportId: videoImport.id } - - const youtubeDL = new YoutubeDLWrapper( - videoImport.targetUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - return processFile( - () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']), - videoImport, - options - ) -} - -async function getVideoImportOrDie (payload: VideoImportPayload) { - const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) - if (!videoImport?.Video) { - throw new Error(`Cannot import video ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`) - } - - return videoImport -} - -type ProcessFileOptions = { - type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType - videoImportId: number -} -async function processFile (downloader: () => Promise, videoImport: MVideoImportDefault, options: ProcessFileOptions) { - let tempVideoPath: string - let videoFile: VideoFileModel - - try { - // Download video from youtubeDL - tempVideoPath = await downloader() - - // Get information about this video - const stats = await stat(tempVideoPath) - const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size) - if (isAble === false) { - throw new Error('The user video quota is exceeded with this video to import.') - } - - const probe = await ffprobePromise(tempVideoPath) - - const { resolution } = await isAudioFile(tempVideoPath, probe) - ? { resolution: VideoResolution.H_NOVIDEO } - : await getVideoStreamDimensionsInfo(tempVideoPath, probe) - - const fps = await getVideoStreamFPS(tempVideoPath, probe) - const duration = await getVideoStreamDuration(tempVideoPath, probe) - - // Prepare video file object for creation in database - const fileExt = getLowercaseExtension(tempVideoPath) - const videoFileData = { - extname: fileExt, - resolution, - size: stats.size, - filename: generateWebVideoFilename(resolution, fileExt), - fps, - videoId: videoImport.videoId - } - videoFile = new VideoFileModel(videoFileData) - - const hookName = options.type === 'youtube-dl' - ? 'filter:api.video.post-import-url.accept.result' - : 'filter:api.video.post-import-torrent.accept.result' - - // Check we accept this video - const acceptParameters = { - videoImport, - video: videoImport.Video, - videoFilePath: tempVideoPath, - videoFile, - user: videoImport.User - } - const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName) - - if (acceptedResult.accepted !== true) { - logger.info('Refused imported video.', { acceptedResult, acceptParameters }) - - videoImport.state = VideoImportState.REJECTED - await videoImport.save() - - throw new Error(acceptedResult.errorMessage) - } - - // Video is accepted, resuming preparation - const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid) - - try { - const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile) - - // Move file - const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) - await move(tempVideoPath, videoDestFile) - - tempVideoPath = null // This path is not used anymore - - let { - miniatureModel: thumbnailModel, - miniatureJSONSave: thumbnailSave - } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE) - - let { - miniatureModel: previewModel, - miniatureJSONSave: previewSave - } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW) - - // Create torrent - await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) - - const videoFileSave = videoFile.toJSON() - - const { videoImportUpdated, video } = await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - // Refresh video - const video = await VideoModel.load(videoImportWithFiles.videoId, t) - if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.') - - await videoFile.save({ transaction: t }) - - // Update video DB object - video.duration = duration - video.state = buildNextVideoState(video.state) - await video.save({ transaction: t }) - - if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) - if (previewModel) await video.addAndSaveThumbnail(previewModel, t) - - // Now we can federate the video (reload from database, we need more attributes) - const videoForFederation = await VideoModel.loadFull(video.uuid, t) - await federateVideoIfNeeded(videoForFederation, true, t) - - // Update video import object - videoImportWithFiles.state = VideoImportState.SUCCESS - const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport - - logger.info('Video %s imported.', video.uuid) - - return { videoImportUpdated, video: videoForFederation } - }).catch(err => { - // Reset fields - if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave) - if (previewModel) previewModel = new ThumbnailModel(previewSave) - - videoFile = new VideoFileModel(videoFileSave) - - throw err - }) - }) - - await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User, videoFileAlreadyLocked: true }) - } finally { - videoFileLockReleaser() - } - } catch (err) { - await onImportError(err, tempVideoPath, videoImport) - - throw err - } -} - -async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise { - // Refresh video, privacy may have changed - const video = await videoImport.Video.reload() - const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) - - return Object.assign(videoImport, { Video: videoWithFiles }) -} - -async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles, videoFile: MVideoFile, thumbnailType: ThumbnailType) { - // Generate miniature if the import did not created it - const needsMiniature = thumbnailType === ThumbnailType.MINIATURE - ? !videoImportWithFiles.Video.getMiniature() - : !videoImportWithFiles.Video.getPreview() - - if (!needsMiniature) { - return { - miniatureModel: null, - miniatureJSONSave: null - } - } - - const miniatureModel = await generateLocalVideoMiniature({ - video: videoImportWithFiles.Video, - videoFile, - type: thumbnailType - }) - const miniatureJSONSave = miniatureModel.toJSON() - - return { - miniatureModel, - miniatureJSONSave - } -} - -async function afterImportSuccess (options: { - videoImport: MVideoImport - video: MVideoFullLight - videoFile: MVideoFile - user: MUserId - videoFileAlreadyLocked: boolean -}) { - const { video, videoFile, videoImport, user, videoFileAlreadyLocked } = options - - Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true }) - - if (video.isBlacklisted()) { - const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video }) - - Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) - } else { - Notifier.Instance.notifyOnNewVideoIfNeeded(video) - } - - // Generate the storyboard in the job queue, and don't forget to federate an update after - await JobQueue.Instance.createJob({ - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - federate: true - } - }) - - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - await JobQueue.Instance.createJob( - await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) - ) - return - } - - if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? - await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user, videoFileAlreadyLocked }) - } -} - -async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) { - try { - if (tempVideoPath) await remove(tempVideoPath) - } catch (errUnlink) { - logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) - } - - videoImport.error = err.message - if (videoImport.state !== VideoImportState.REJECTED) { - videoImport.state = VideoImportState.FAILED - } - await videoImport.save() - - Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false }) -} diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts deleted file mode 100644 index 070d1d7a2..000000000 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ /dev/null @@ -1,279 +0,0 @@ -import { Job } from 'bullmq' -import { readdir, remove } from 'fs-extra' -import { join } from 'path' -import { peertubeTruncate } from '@server/helpers/core-utils' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' -import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' -import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail' -import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { moveToNextState } from '@server/lib/video-state' -import { VideoModel } from '@server/models/video/video' -import { VideoBlacklistModel } from '@server/models/video/video-blacklist' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' -import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' -import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { JobQueue } from '../job-queue' - -const lTags = loggerTagsFactory('live', 'job') - -async function processVideoLiveEnding (job: Job) { - const payload = job.data as VideoLiveEndingPayload - - logger.info('Processing video live ending for %s.', payload.videoId, { payload, ...lTags() }) - - function logError () { - logger.warn('Video live %d does not exist anymore. Cannot process live ending.', payload.videoId, lTags()) - } - - const video = await VideoModel.load(payload.videoId) - const live = await VideoLiveModel.loadByVideoId(payload.videoId) - const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) - - if (!video || !live || !liveSession) { - logError() - return - } - - const permanentLive = live.permanentLive - - liveSession.endingProcessed = true - await liveSession.save() - - if (liveSession.saveReplay !== true) { - return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) - } - - if (permanentLive) { - await saveReplayToExternalVideo({ - liveVideo: video, - liveSession, - publishedAt: payload.publishedAt, - replayDirectory: payload.replayDirectory - }) - - return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) - } - - return replaceLiveByReplay({ - video, - liveSession, - live, - permanentLive, - replayDirectory: payload.replayDirectory - }) -} - -// --------------------------------------------------------------------------- - -export { - processVideoLiveEnding -} - -// --------------------------------------------------------------------------- - -async function saveReplayToExternalVideo (options: { - liveVideo: MVideo - liveSession: MVideoLiveSession - publishedAt: string - replayDirectory: string -}) { - const { liveVideo, liveSession, publishedAt, replayDirectory } = options - - const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) - - const videoNameSuffix = ` - ${new Date(publishedAt).toLocaleString()}` - const truncatedVideoName = peertubeTruncate(liveVideo.name, { - length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max - videoNameSuffix.length - }) - - const replayVideo = new VideoModel({ - name: truncatedVideoName + videoNameSuffix, - isLive: false, - state: VideoState.TO_TRANSCODE, - duration: 0, - - remote: liveVideo.remote, - category: liveVideo.category, - licence: liveVideo.licence, - language: liveVideo.language, - commentsEnabled: liveVideo.commentsEnabled, - downloadEnabled: liveVideo.downloadEnabled, - waitTranscoding: true, - nsfw: liveVideo.nsfw, - description: liveVideo.description, - support: liveVideo.support, - privacy: replaySettings.privacy, - channelId: liveVideo.channelId - }) as MVideoWithAllFiles - - replayVideo.Thumbnails = [] - replayVideo.VideoFiles = [] - replayVideo.VideoStreamingPlaylists = [] - - replayVideo.url = getLocalVideoActivityPubUrl(replayVideo) - - await replayVideo.save() - - liveSession.replayVideoId = replayVideo.id - await liveSession.save() - - // If live is blacklisted, also blacklist the replay - const blacklist = await VideoBlacklistModel.loadByVideoId(liveVideo.id) - if (blacklist) { - await VideoBlacklistModel.create({ - videoId: replayVideo.id, - unfederated: blacklist.unfederated, - reason: blacklist.reason, - type: blacklist.type - }) - } - - await assignReplayFilesToVideo({ video: replayVideo, replayDirectory }) - - await remove(replayDirectory) - - for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { - const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) - await replayVideo.addAndSaveThumbnail(image) - } - - await moveToNextState({ video: replayVideo, isNewVideo: true }) - - await createStoryboardJob(replayVideo) -} - -async function replaceLiveByReplay (options: { - video: MVideo - liveSession: MVideoLiveSession - live: MVideoLive - permanentLive: boolean - replayDirectory: string -}) { - const { video, liveSession, live, permanentLive, replayDirectory } = options - - const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) - const videoWithFiles = await VideoModel.loadFull(video.id) - const hlsPlaylist = videoWithFiles.getHLSPlaylist() - - await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist) - - await live.destroy() - - videoWithFiles.isLive = false - videoWithFiles.privacy = replaySettings.privacy - videoWithFiles.waitTranscoding = true - videoWithFiles.state = VideoState.TO_TRANSCODE - - await videoWithFiles.save() - - liveSession.replayVideoId = videoWithFiles.id - await liveSession.save() - - await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) - - // Reset playlist - hlsPlaylist.VideoFiles = [] - hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename() - hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename() - await hlsPlaylist.save() - - await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) - - // Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay - if (permanentLive) { // Remove session replay - await remove(replayDirectory) - } else { // We won't stream again in this live, we can delete the base replay directory - await remove(getLiveReplayBaseDirectory(videoWithFiles)) - } - - // Regenerate the thumbnail & preview? - await regenerateMiniaturesIfNeeded(videoWithFiles) - - // We consider this is a new video - await moveToNextState({ video: videoWithFiles, isNewVideo: true }) - - await createStoryboardJob(videoWithFiles) -} - -async function assignReplayFilesToVideo (options: { - video: MVideo - replayDirectory: string -}) { - const { video, replayDirectory } = options - - const concatenatedTsFiles = await readdir(replayDirectory) - - for (const concatenatedTsFile of concatenatedTsFiles) { - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - await video.reload() - - const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) - - const probe = await ffprobePromise(concatenatedTsFilePath) - const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) - const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) - const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe) - - try { - await generateHlsPlaylistResolutionFromTS({ - video, - inputFileMutexReleaser, - concatenatedTsFilePath, - resolution, - fps, - isAAC: audioStream?.codec_name === 'aac' - }) - } catch (err) { - logger.error('Cannot generate HLS playlist resolution from TS files.', { err }) - } - - inputFileMutexReleaser() - } - - return video -} - -async function cleanupLiveAndFederate (options: { - video: MVideo - permanentLive: boolean - streamingPlaylistId: number -}) { - const { permanentLive, video, streamingPlaylistId } = options - - const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) - - if (streamingPlaylist) { - if (permanentLive) { - await cleanupAndDestroyPermanentLive(video, streamingPlaylist) - } else { - await cleanupUnsavedNormalLive(video, streamingPlaylist) - } - } - - try { - const fullVideo = await VideoModel.loadFull(video.id) - return federateVideoIfNeeded(fullVideo, false, undefined) - } catch (err) { - logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) - } -} - -function createStoryboardJob (video: MVideo) { - return JobQueue.Instance.createJob({ - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - federate: true - } - }) -} diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts deleted file mode 100644 index bac99fdb7..000000000 --- a/server/lib/job-queue/handlers/video-redundancy.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Job } from 'bullmq' -import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' -import { VideoRedundancyPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processVideoRedundancy (job: Job) { - const payload = job.data as VideoRedundancyPayload - logger.info('Processing video redundancy in job %s.', job.id) - - return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) -} - -// --------------------------------------------------------------------------- - -export { - processVideoRedundancy -} diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts deleted file mode 100644 index caf051bfa..000000000 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { Job } from 'bullmq' -import { remove } from 'fs-extra' -import { join } from 'path' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { CONFIG } from '@server/initializers/config' -import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' -import { isAbleToUploadVideo } from '@server/lib/user' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { MVideo, MVideoFullLight } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { FFmpegEdition } from '@shared/ffmpeg' -import { - VideoStudioEditionPayload, - VideoStudioTask, - VideoStudioTaskCutPayload, - VideoStudioTaskIntroPayload, - VideoStudioTaskOutroPayload, - VideoStudioTaskPayload, - VideoStudioTaskWatermarkPayload -} from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' - -const lTagsBase = loggerTagsFactory('video-studio') - -async function processVideoStudioEdition (job: Job) { - const payload = job.data as VideoStudioEditionPayload - const lTags = lTagsBase(payload.videoUUID) - - logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) - - try { - const video = await VideoModel.loadFull(payload.videoUUID) - - // No video, maybe deleted? - if (!video) { - logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) - - await safeCleanupStudioTMPFiles(payload.tasks) - return undefined - } - - await checkUserQuotaOrThrow(video, payload) - - const inputFile = video.getMaxQualityFile() - - const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { - let tmpInputFilePath: string - let outputPath: string - - for (const task of payload.tasks) { - const outputFilename = buildUUID() + inputFile.extname - outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) - - await processTask({ - inputPath: tmpInputFilePath ?? originalFilePath, - video, - outputPath, - task, - lTags - }) - - if (tmpInputFilePath) await remove(tmpInputFilePath) - - // For the next iteration - tmpInputFilePath = outputPath - } - - return outputPath - }) - - logger.info('Video edition ended for video %s.', video.uuid, lTags) - - await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks }) - } catch (err) { - await safeCleanupStudioTMPFiles(payload.tasks) - - throw err - } -} - -// --------------------------------------------------------------------------- - -export { - processVideoStudioEdition -} - -// --------------------------------------------------------------------------- - -type TaskProcessorOptions = { - inputPath: string - outputPath: string - video: MVideo - task: T - lTags: { tags: string[] } -} - -const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise } = { - 'add-intro': processAddIntroOutro, - 'add-outro': processAddIntroOutro, - 'cut': processCut, - 'add-watermark': processAddWatermark -} - -async function processTask (options: TaskProcessorOptions) { - const { video, task, lTags } = options - - logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags }) - - const processor = taskProcessors[options.task.name] - if (!process) throw new Error('Unknown task ' + task.name) - - return processor(options) -} - -function processAddIntroOutro (options: TaskProcessorOptions) { - const { task, lTags } = options - - logger.debug('Will add intro/outro to the video.', { options, ...lTags }) - - return buildFFmpegEdition().addIntroOutro({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - introOutroPath: task.options.file, - type: task.name === 'add-intro' - ? 'intro' - : 'outro' - }) -} - -function processCut (options: TaskProcessorOptions) { - const { task, lTags } = options - - logger.debug('Will cut the video.', { options, ...lTags }) - - return buildFFmpegEdition().cutVideo({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - start: task.options.start, - end: task.options.end - }) -} - -function processAddWatermark (options: TaskProcessorOptions) { - const { task, lTags } = options - - logger.debug('Will add watermark to the video.', { options, ...lTags }) - - return buildFFmpegEdition().addWatermark({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - watermarkPath: task.options.file, - - videoFilters: { - watermarkSizeRatio: task.options.watermarkSizeRatio, - horitonzalMarginRatio: task.options.horitonzalMarginRatio, - verticalMarginRatio: task.options.verticalMarginRatio - } - }) -} - -// --------------------------------------------------------------------------- - -async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) { - const user = await UserModel.loadByVideoId(video.id) - - const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file - - const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder) - if (await isAbleToUploadVideo(user.id, additionalBytes) === false) { - throw new Error('Quota exceeded for this user to edit the video') - } -} - -function buildFFmpegEdition () { - return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) -} diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts deleted file mode 100644 index 1c8f4fd9f..000000000 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { Job } from 'bullmq' -import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' -import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding' -import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding' -import { removeAllWebVideoFiles } from '@server/lib/video-file' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { moveToFailedTranscodingState } from '@server/lib/video-state' -import { UserModel } from '@server/models/user/user' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MUser, MUserId, MVideoFullLight } from '@server/types/models' -import { - HLSTranscodingPayload, - MergeAudioTranscodingPayload, - NewWebVideoResolutionTranscodingPayload, - OptimizeTranscodingPayload, - VideoTranscodingPayload -} from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' - -type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise - -const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { - 'new-resolution-to-hls': handleHLSJob, - 'new-resolution-to-web-video': handleNewWebVideoResolutionJob, - 'merge-audio-to-web-video': handleWebVideoMergeAudioJob, - 'optimize-to-web-video': handleWebVideoOptimizeJob -} - -const lTags = loggerTagsFactory('transcoding') - -async function processVideoTranscoding (job: Job) { - const payload = job.data as VideoTranscodingPayload - logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID)) - - const video = await VideoModel.loadFull(payload.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Do not process job %d, video does not exist.', job.id, lTags(payload.videoUUID)) - return undefined - } - - const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) - - const handler = handlers[payload.type] - - if (!handler) { - await moveToFailedTranscodingState(video) - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - throw new Error('Cannot find transcoding handler for ' + payload.type) - } - - try { - await handler(job, payload, video, user) - } catch (error) { - await moveToFailedTranscodingState(video) - - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - throw error - } - - return video -} - -// --------------------------------------------------------------------------- - -export { - processVideoTranscoding -} - -// --------------------------------------------------------------------------- -// Job handlers -// --------------------------------------------------------------------------- - -async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { - logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) - - await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) - - logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) -} - -async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { - logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) - - await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) - - logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) -} - -// --------------------------------------------------------------------------- - -async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) { - logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) - - await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) - - logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) -} - -// --------------------------------------------------------------------------- - -async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFullLight) { - logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload }) - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) - let video: MVideoFullLight - - try { - video = await VideoModel.loadFull(videoArg.uuid) - - const videoFileInput = payload.copyCodecs - ? video.getWebVideoFile(payload.resolution) - : video.getMaxQualityFile() - - const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { - return generateHlsPlaylistResolution({ - video, - videoInputPath, - inputFileMutexReleaser, - resolution: payload.resolution, - fps: payload.fps, - copyCodecs: payload.copyCodecs, - job - }) - }) - } finally { - inputFileMutexReleaser() - } - - logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - if (payload.deleteWebVideoFiles === true) { - logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) - - await removeAllWebVideoFiles(video) - } - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) -} diff --git a/server/lib/job-queue/handlers/video-views-stats.ts b/server/lib/job-queue/handlers/video-views-stats.ts deleted file mode 100644 index c9aa218e5..000000000 --- a/server/lib/job-queue/handlers/video-views-stats.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { VideoViewModel } from '@server/models/view/video-view' -import { isTestOrDevInstance } from '../../../helpers/core-utils' -import { logger } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' -import { Redis } from '../../redis' - -async function processVideosViewsStats () { - const lastHour = new Date() - - // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour - if (!isTestOrDevInstance()) lastHour.setHours(lastHour.getHours() - 1) - - const hour = lastHour.getHours() - const startDate = lastHour.setMinutes(0, 0, 0) - const endDate = lastHour.setMinutes(59, 59, 999) - - const videoIds = await Redis.Instance.listVideosViewedForStats(hour) - if (videoIds.length === 0) return - - logger.info('Processing videos views stats in job for hour %d.', hour) - - for (const videoId of videoIds) { - try { - const views = await Redis.Instance.getVideoViewsStats(videoId, hour) - await Redis.Instance.deleteVideoViewsStats(videoId, hour) - - if (views) { - logger.debug('Adding %d views to video %d stats in hour %d.', views, videoId, hour) - - try { - const video = await VideoModel.load(videoId) - if (!video) { - logger.debug('Video %d does not exist anymore, skipping videos view stats.', videoId) - continue - } - - await VideoViewModel.create({ - startDate: new Date(startDate), - endDate: new Date(endDate), - views, - videoId - }) - } catch (err) { - logger.error('Cannot create video views stats for video %d in hour %d.', videoId, hour, { err }) - } - } - } catch (err) { - logger.error('Cannot update video views stats of video %d in hour %d.', videoId, hour, { err }) - } - } -} - -// --------------------------------------------------------------------------- - -export { - processVideosViewsStats -} diff --git a/server/lib/job-queue/index.ts b/server/lib/job-queue/index.ts deleted file mode 100644 index 57231e649..000000000 --- a/server/lib/job-queue/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './job-queue' diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts deleted file mode 100644 index 177bca285..000000000 --- a/server/lib/job-queue/job-queue.ts +++ /dev/null @@ -1,537 +0,0 @@ -import { - FlowJob, - FlowProducer, - Job, - JobsOptions, - Queue, - QueueEvents, - QueueEventsOptions, - QueueOptions, - Worker, - WorkerOptions -} from 'bullmq' -import { parseDurationToMs } from '@server/helpers/core-utils' -import { jobStates } from '@server/helpers/custom-validators/jobs' -import { CONFIG } from '@server/initializers/config' -import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy' -import { pick, timeoutPromise } from '@shared/core-utils' -import { - ActivitypubFollowPayload, - ActivitypubHttpBroadcastPayload, - ActivitypubHttpFetcherPayload, - ActivitypubHttpUnicastPayload, - ActorKeysPayload, - AfterVideoChannelImportPayload, - DeleteResumableUploadMetaFilePayload, - EmailPayload, - FederateVideoPayload, - GenerateStoryboardPayload, - JobState, - JobType, - ManageVideoTorrentPayload, - MoveObjectStoragePayload, - NotifyPayload, - RefreshPayload, - TranscodingJobBuilderPayload, - VideoChannelImportPayload, - VideoFileImportPayload, - VideoImportPayload, - VideoLiveEndingPayload, - VideoRedundancyPayload, - VideoStudioEditionPayload, - VideoTranscodingPayload -} from '../../../shared/models' -import { logger } from '../../helpers/logger' -import { JOB_ATTEMPTS, JOB_CONCURRENCY, JOB_REMOVAL_OPTIONS, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' -import { Hooks } from '../plugins/hooks' -import { Redis } from '../redis' -import { processActivityPubCleaner } from './handlers/activitypub-cleaner' -import { processActivityPubFollow } from './handlers/activitypub-follow' -import { processActivityPubHttpSequentialBroadcast, processActivityPubParallelHttpBroadcast } from './handlers/activitypub-http-broadcast' -import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' -import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' -import { refreshAPObject } from './handlers/activitypub-refresher' -import { processActorKeys } from './handlers/actor-keys' -import { processAfterVideoChannelImport } from './handlers/after-video-channel-import' -import { processEmail } from './handlers/email' -import { processFederateVideo } from './handlers/federate-video' -import { processManageVideoTorrent } from './handlers/manage-video-torrent' -import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' -import { processNotify } from './handlers/notify' -import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder' -import { processVideoChannelImport } from './handlers/video-channel-import' -import { processVideoFileImport } from './handlers/video-file-import' -import { processVideoImport } from './handlers/video-import' -import { processVideoLiveEnding } from './handlers/video-live-ending' -import { processVideoStudioEdition } from './handlers/video-studio-edition' -import { processVideoTranscoding } from './handlers/video-transcoding' -import { processVideosViewsStats } from './handlers/video-views-stats' -import { processGenerateStoryboard } from './handlers/generate-storyboard' - -export type CreateJobArgument = - { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | - { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | - { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | - { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | - { type: 'activitypub-cleaner', payload: {} } | - { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | - { type: 'video-file-import', payload: VideoFileImportPayload } | - { type: 'video-transcoding', payload: VideoTranscodingPayload } | - { type: 'email', payload: EmailPayload } | - { type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } | - { type: 'video-import', payload: VideoImportPayload } | - { type: 'activitypub-refresher', payload: RefreshPayload } | - { type: 'videos-views-stats', payload: {} } | - { type: 'video-live-ending', payload: VideoLiveEndingPayload } | - { type: 'actor-keys', payload: ActorKeysPayload } | - { type: 'video-redundancy', payload: VideoRedundancyPayload } | - { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | - { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | - { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | - { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | - { type: 'video-channel-import', payload: VideoChannelImportPayload } | - { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | - { type: 'notify', payload: NotifyPayload } | - { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | - { type: 'federate-video', payload: FederateVideoPayload } | - { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } - -export type CreateJobOptions = { - delay?: number - priority?: number - failParentOnFailure?: boolean -} - -const handlers: { [id in JobType]: (job: Job) => Promise } = { - 'activitypub-cleaner': processActivityPubCleaner, - 'activitypub-follow': processActivityPubFollow, - 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, - 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, - 'activitypub-http-fetcher': processActivityPubHttpFetcher, - 'activitypub-http-unicast': processActivityPubHttpUnicast, - 'activitypub-refresher': refreshAPObject, - 'actor-keys': processActorKeys, - 'after-video-channel-import': processAfterVideoChannelImport, - 'email': processEmail, - 'federate-video': processFederateVideo, - 'transcoding-job-builder': processTranscodingJobBuilder, - 'manage-video-torrent': processManageVideoTorrent, - 'move-to-object-storage': processMoveToObjectStorage, - 'notify': processNotify, - 'video-channel-import': processVideoChannelImport, - 'video-file-import': processVideoFileImport, - 'video-import': processVideoImport, - 'video-live-ending': processVideoLiveEnding, - 'video-redundancy': processVideoRedundancy, - 'video-studio-edition': processVideoStudioEdition, - 'video-transcoding': processVideoTranscoding, - 'videos-views-stats': processVideosViewsStats, - 'generate-video-storyboard': processGenerateStoryboard -} - -const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = { - 'move-to-object-storage': onMoveToObjectStorageFailure -} - -const jobTypes: JobType[] = [ - 'activitypub-cleaner', - 'activitypub-follow', - 'activitypub-http-broadcast-parallel', - 'activitypub-http-broadcast', - 'activitypub-http-fetcher', - 'activitypub-http-unicast', - 'activitypub-refresher', - 'actor-keys', - 'after-video-channel-import', - 'email', - 'federate-video', - 'generate-video-storyboard', - 'manage-video-torrent', - 'move-to-object-storage', - 'notify', - 'transcoding-job-builder', - 'video-channel-import', - 'video-file-import', - 'video-import', - 'video-live-ending', - 'video-redundancy', - 'video-studio-edition', - 'video-transcoding', - 'videos-views-stats' -] - -const silentFailure = new Set([ 'activitypub-http-unicast' ]) - -class JobQueue { - - private static instance: JobQueue - - private workers: { [id in JobType]?: Worker } = {} - private queues: { [id in JobType]?: Queue } = {} - private queueEvents: { [id in JobType]?: QueueEvents } = {} - - private flowProducer: FlowProducer - - private initialized = false - private jobRedisPrefix: string - - private constructor () { - } - - init () { - // Already initialized - if (this.initialized === true) return - this.initialized = true - - this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST - - for (const handlerName of Object.keys(handlers)) { - this.buildWorker(handlerName) - this.buildQueue(handlerName) - this.buildQueueEvent(handlerName) - } - - this.flowProducer = new FlowProducer({ - connection: Redis.getRedisClientOptions('FlowProducer'), - prefix: this.jobRedisPrefix - }) - this.flowProducer.on('error', err => { logger.error('Error in flow producer', { err }) }) - - this.addRepeatableJobs() - } - - private buildWorker (handlerName: JobType) { - const workerOptions: WorkerOptions = { - autorun: false, - concurrency: this.getJobConcurrency(handlerName), - prefix: this.jobRedisPrefix, - connection: Redis.getRedisClientOptions('Worker'), - maxStalledCount: 10 - } - - const handler = function (job: Job) { - const timeout = JOB_TTL[handlerName] - const p = handlers[handlerName](job) - - if (!timeout) return p - - return timeoutPromise(p, timeout) - } - - const processor = async (jobArg: Job) => { - const job = await Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName }) - - return Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result') - } - - const worker = new Worker(handlerName, processor, workerOptions) - - worker.on('failed', (job, err) => { - const logLevel = silentFailure.has(handlerName) - ? 'debug' - : 'error' - - logger.log(logLevel, 'Cannot execute job %s in queue %s.', job.id, handlerName, { payload: job.data, err }) - - if (errorHandlers[job.name]) { - errorHandlers[job.name](job, err) - .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err })) - } - }) - - worker.on('error', err => { logger.error('Error in job worker %s.', handlerName, { err }) }) - - this.workers[handlerName] = worker - } - - private buildQueue (handlerName: JobType) { - const queueOptions: QueueOptions = { - connection: Redis.getRedisClientOptions('Queue'), - prefix: this.jobRedisPrefix - } - - const queue = new Queue(handlerName, queueOptions) - queue.on('error', err => { logger.error('Error in job queue %s.', handlerName, { err }) }) - - this.queues[handlerName] = queue - } - - private buildQueueEvent (handlerName: JobType) { - const queueEventsOptions: QueueEventsOptions = { - autorun: false, - connection: Redis.getRedisClientOptions('QueueEvent'), - prefix: this.jobRedisPrefix - } - - const queueEvents = new QueueEvents(handlerName, queueEventsOptions) - queueEvents.on('error', err => { logger.error('Error in job queue events %s.', handlerName, { err }) }) - - this.queueEvents[handlerName] = queueEvents - } - - // --------------------------------------------------------------------------- - - async terminate () { - const promises = Object.keys(this.workers) - .map(handlerName => { - const worker: Worker = this.workers[handlerName] - const queue: Queue = this.queues[handlerName] - const queueEvent: QueueEvents = this.queueEvents[handlerName] - - return Promise.all([ - worker.close(false), - queue.close(), - queueEvent.close() - ]) - }) - - return Promise.all(promises) - } - - start () { - const promises = Object.keys(this.workers) - .map(handlerName => { - const worker: Worker = this.workers[handlerName] - const queueEvent: QueueEvents = this.queueEvents[handlerName] - - return Promise.all([ - worker.run(), - queueEvent.run() - ]) - }) - - return Promise.all(promises) - } - - async pause () { - for (const handlerName of Object.keys(this.workers)) { - const worker: Worker = this.workers[handlerName] - - await worker.pause() - } - } - - resume () { - for (const handlerName of Object.keys(this.workers)) { - const worker: Worker = this.workers[handlerName] - - worker.resume() - } - } - - // --------------------------------------------------------------------------- - - createJobAsync (options: CreateJobArgument & CreateJobOptions): void { - this.createJob(options) - .catch(err => logger.error('Cannot create job.', { err, options })) - } - - createJob (options: CreateJobArgument & CreateJobOptions) { - const queue: Queue = this.queues[options.type] - if (queue === undefined) { - logger.error('Unknown queue %s: cannot create job.', options.type) - return - } - - const jobOptions = this.buildJobOptions(options.type as JobType, pick(options, [ 'priority', 'delay' ])) - - return queue.add('job', options.payload, jobOptions) - } - - createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { - let lastJob: FlowJob - - for (const job of jobs) { - if (!job) continue - - lastJob = { - ...this.buildJobFlowOption(job), - - children: lastJob - ? [ lastJob ] - : [] - } - } - - return this.flowProducer.add(lastJob) - } - - createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { - return this.flowProducer.add({ - ...this.buildJobFlowOption(parent), - - children: children.map(c => this.buildJobFlowOption(c)) - }) - } - - private buildJobFlowOption (job: CreateJobArgument & CreateJobOptions): FlowJob { - return { - name: 'job', - data: job.payload, - queueName: job.type, - opts: { - failParentOnFailure: true, - - ...this.buildJobOptions(job.type as JobType, pick(job, [ 'priority', 'delay', 'failParentOnFailure' ])) - } - } - } - - private buildJobOptions (type: JobType, options: CreateJobOptions = {}): JobsOptions { - return { - backoff: { delay: 60 * 1000, type: 'exponential' }, - attempts: JOB_ATTEMPTS[type], - priority: options.priority, - delay: options.delay, - - ...this.buildJobRemovalOptions(type) - } - } - - // --------------------------------------------------------------------------- - - async listForApi (options: { - state?: JobState - start: number - count: number - asc?: boolean - jobType: JobType - }): Promise { - const { state, start, count, asc, jobType } = options - - const states = this.buildStateFilter(state) - const filteredJobTypes = this.buildTypeFilter(jobType) - - let results: Job[] = [] - - for (const jobType of filteredJobTypes) { - const queue: Queue = this.queues[jobType] - - if (queue === undefined) { - logger.error('Unknown queue %s to list jobs.', jobType) - continue - } - - const jobs = await queue.getJobs(states, 0, start + count, asc) - results = results.concat(jobs) - } - - results.sort((j1: any, j2: any) => { - if (j1.timestamp < j2.timestamp) return -1 - else if (j1.timestamp === j2.timestamp) return 0 - - return 1 - }) - - if (asc === false) results.reverse() - - return results.slice(start, start + count) - } - - async count (state: JobState, jobType?: JobType): Promise { - const states = state ? [ state ] : jobStates - const filteredJobTypes = this.buildTypeFilter(jobType) - - let total = 0 - - for (const type of filteredJobTypes) { - const queue = this.queues[type] - if (queue === undefined) { - logger.error('Unknown queue %s to count jobs.', type) - continue - } - - const counts = await queue.getJobCounts() - - for (const s of states) { - total += counts[s] - } - } - - return total - } - - private buildStateFilter (state?: JobState) { - if (!state) return jobStates - - const states = [ state ] - - // Include parent if filtering on waiting - if (state === 'waiting') states.push('waiting-children') - - return states - } - - private buildTypeFilter (jobType?: JobType) { - if (!jobType) return jobTypes - - return jobTypes.filter(t => t === jobType) - } - - async getStats () { - const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() })) - - return Promise.all(promises) - } - - // --------------------------------------------------------------------------- - - async removeOldJobs () { - for (const key of Object.keys(this.queues)) { - const queue: Queue = this.queues[key] - await queue.clean(parseDurationToMs('7 days'), 1000, 'completed') - await queue.clean(parseDurationToMs('7 days'), 1000, 'failed') - } - } - - private addRepeatableJobs () { - this.queues['videos-views-stats'].add('job', {}, { - repeat: REPEAT_JOBS['videos-views-stats'], - - ...this.buildJobRemovalOptions('videos-views-stats') - }).catch(err => logger.error('Cannot add repeatable job.', { err })) - - if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) { - this.queues['activitypub-cleaner'].add('job', {}, { - repeat: REPEAT_JOBS['activitypub-cleaner'], - - ...this.buildJobRemovalOptions('activitypub-cleaner') - }).catch(err => logger.error('Cannot add repeatable job.', { err })) - } - } - - private getJobConcurrency (jobType: JobType) { - if (jobType === 'video-transcoding') return CONFIG.TRANSCODING.CONCURRENCY - if (jobType === 'video-import') return CONFIG.IMPORT.VIDEOS.CONCURRENCY - - return JOB_CONCURRENCY[jobType] - } - - private buildJobRemovalOptions (queueName: string) { - return { - removeOnComplete: { - // Wants seconds - age: (JOB_REMOVAL_OPTIONS.SUCCESS[queueName] || JOB_REMOVAL_OPTIONS.SUCCESS.DEFAULT) / 1000, - - count: JOB_REMOVAL_OPTIONS.COUNT - }, - removeOnFail: { - // Wants seconds - age: (JOB_REMOVAL_OPTIONS.FAILURE[queueName] || JOB_REMOVAL_OPTIONS.FAILURE.DEFAULT) / 1000, - - count: JOB_REMOVAL_OPTIONS.COUNT / 1000 - } - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - jobTypes, - JobQueue -} diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts deleted file mode 100644 index 8b46800da..000000000 --- a/server/lib/live/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './live-manager' -export * from './live-quota-store' -export * from './live-segment-sha-store' -export * from './live-utils' diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts deleted file mode 100644 index acb7af274..000000000 --- a/server/lib/live/live-manager.ts +++ /dev/null @@ -1,552 +0,0 @@ -import { readdir, readFile } from 'fs-extra' -import { createServer, Server } from 'net' -import { join } from 'path' -import { createServer as createServerTLS, Server as ServerTLS } from 'tls' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' -import { VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models' -import { pick, wait } from '@shared/core-utils' -import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg' -import { LiveVideoError, VideoState } from '@shared/models' -import { federateVideoIfNeeded } from '../activitypub/videos' -import { JobQueue } from '../job-queue' -import { getLiveReplayBaseDirectory } from '../paths' -import { PeerTubeSocket } from '../peertube-socket' -import { Hooks } from '../plugins/hooks' -import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions' -import { LiveQuotaStore } from './live-quota-store' -import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils' -import { MuxingSession } from './shared' - -const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') -const context = require('node-media-server/src/node_core_ctx') -const nodeMediaServerLogger = require('node-media-server/src/node_core_logger') - -// Disable node media server logs -nodeMediaServerLogger.setLogType(0) - -const config = { - rtmp: { - port: CONFIG.LIVE.RTMP.PORT, - chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, - gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, - ping: VIDEO_LIVE.RTMP.PING, - ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT - } -} - -const lTags = loggerTagsFactory('live') - -class LiveManager { - - private static instance: LiveManager - - private readonly muxingSessions = new Map() - private readonly videoSessions = new Map() - - private rtmpServer: Server - private rtmpsServer: ServerTLS - - private running = false - - private constructor () { - } - - init () { - const events = this.getContext().nodeEvent - events.on('postPublish', (sessionId: string, streamPath: string) => { - logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) - - const splittedPath = streamPath.split('/') - if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { - logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) - return this.abortSession(sessionId) - } - - const session = this.getContext().sessions.get(sessionId) - const inputLocalUrl = session.inputOriginLocalUrl + streamPath - const inputPublicUrl = session.inputOriginPublicUrl + streamPath - - this.handleSession({ sessionId, inputPublicUrl, inputLocalUrl, streamKey: splittedPath[2] }) - .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) - }) - - events.on('donePublish', sessionId => { - logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) - - // Force session aborting, so we kill ffmpeg even if it still has data to process (slow CPU) - setTimeout(() => this.abortSession(sessionId), 2000) - }) - - registerConfigChangedHandler(() => { - if (!this.running && CONFIG.LIVE.ENABLED === true) { - this.run().catch(err => logger.error('Cannot run live server.', { err })) - return - } - - if (this.running && CONFIG.LIVE.ENABLED === false) { - this.stop() - } - }) - - // Cleanup broken lives, that were terminated by a server restart for example - this.handleBrokenLives() - .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) - } - - async run () { - this.running = true - - if (CONFIG.LIVE.RTMP.ENABLED) { - logger.info('Running RTMP server on port %d', CONFIG.LIVE.RTMP.PORT, lTags()) - - this.rtmpServer = createServer(socket => { - const session = new NodeRtmpSession(config, socket) - - session.inputOriginLocalUrl = 'rtmp://127.0.0.1:' + CONFIG.LIVE.RTMP.PORT - session.inputOriginPublicUrl = WEBSERVER.RTMP_URL - session.run() - }) - - this.rtmpServer.on('error', err => { - logger.error('Cannot run RTMP server.', { err, ...lTags() }) - }) - - this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT, CONFIG.LIVE.RTMP.HOSTNAME) - } - - if (CONFIG.LIVE.RTMPS.ENABLED) { - logger.info('Running RTMPS server on port %d', CONFIG.LIVE.RTMPS.PORT, lTags()) - - const [ key, cert ] = await Promise.all([ - readFile(CONFIG.LIVE.RTMPS.KEY_FILE), - readFile(CONFIG.LIVE.RTMPS.CERT_FILE) - ]) - const serverOptions = { key, cert } - - this.rtmpsServer = createServerTLS(serverOptions, socket => { - const session = new NodeRtmpSession(config, socket) - - session.inputOriginLocalUrl = 'rtmps://127.0.0.1:' + CONFIG.LIVE.RTMPS.PORT - session.inputOriginPublicUrl = WEBSERVER.RTMPS_URL - session.run() - }) - - this.rtmpsServer.on('error', err => { - logger.error('Cannot run RTMPS server.', { err, ...lTags() }) - }) - - this.rtmpsServer.listen(CONFIG.LIVE.RTMPS.PORT, CONFIG.LIVE.RTMPS.HOSTNAME) - } - } - - stop () { - this.running = false - - if (this.rtmpServer) { - logger.info('Stopping RTMP server.', lTags()) - - this.rtmpServer.close() - this.rtmpServer = undefined - } - - if (this.rtmpsServer) { - logger.info('Stopping RTMPS server.', lTags()) - - this.rtmpsServer.close() - this.rtmpsServer = undefined - } - - // Sessions is an object - this.getContext().sessions.forEach((session: any) => { - if (session instanceof NodeRtmpSession) { - session.stop() - } - }) - } - - isRunning () { - return !!this.rtmpServer - } - - hasSession (sessionId: string) { - return this.getContext().sessions.has(sessionId) - } - - stopSessionOf (videoUUID: string, error: LiveVideoError | null) { - const sessionId = this.videoSessions.get(videoUUID) - if (!sessionId) { - logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID)) - return - } - - logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) }) - - this.saveEndingSession(videoUUID, error) - .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) })) - - this.videoSessions.delete(videoUUID) - this.abortSession(sessionId) - } - - private getContext () { - return context - } - - private abortSession (sessionId: string) { - const session = this.getContext().sessions.get(sessionId) - if (session) { - session.stop() - this.getContext().sessions.delete(sessionId) - } - - const muxingSession = this.muxingSessions.get(sessionId) - if (muxingSession) { - // Muxing session will fire and event so we correctly cleanup the session - muxingSession.abort() - - this.muxingSessions.delete(sessionId) - } - } - - private async handleSession (options: { - sessionId: string - inputLocalUrl: string - inputPublicUrl: string - streamKey: string - }) { - const { inputLocalUrl, inputPublicUrl, sessionId, streamKey } = options - - const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) - if (!videoLive) { - logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) - return this.abortSession(sessionId) - } - - const video = videoLive.Video - if (video.isBlacklisted()) { - logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) - return this.abortSession(sessionId) - } - - if (this.videoSessions.has(video.uuid)) { - logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid)) - return this.abortSession(sessionId) - } - - // Cleanup old potential live (could happen with a permanent live) - const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) - if (oldStreamingPlaylist) { - if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) - - await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) - } - - this.videoSessions.set(video.uuid, sessionId) - - const now = Date.now() - const probe = await ffprobePromise(inputLocalUrl) - - const [ { resolution, ratio }, fps, bitrate, hasAudio ] = await Promise.all([ - getVideoStreamDimensionsInfo(inputLocalUrl, probe), - getVideoStreamFPS(inputLocalUrl, probe), - getVideoStreamBitrate(inputLocalUrl, probe), - hasAudioStream(inputLocalUrl, probe) - ]) - - logger.info( - '%s probing took %d ms (bitrate: %d, fps: %d, resolution: %d)', - inputLocalUrl, Date.now() - now, bitrate, fps, resolution, lTags(sessionId, video.uuid) - ) - - const allResolutions = await Hooks.wrapObject( - this.buildAllResolutionsToTranscode(resolution, hasAudio), - 'filter:transcoding.auto.resolutions-to-transcode.result', - { video } - ) - - logger.info( - 'Handling live video of original resolution %d.', resolution, - { allResolutions, ...lTags(sessionId, video.uuid) } - ) - - return this.runMuxingSession({ - sessionId, - videoLive, - - inputLocalUrl, - inputPublicUrl, - fps, - bitrate, - ratio, - allResolutions, - hasAudio - }) - } - - private async runMuxingSession (options: { - sessionId: string - videoLive: MVideoLiveVideoWithSetting - - inputLocalUrl: string - inputPublicUrl: string - - fps: number - bitrate: number - ratio: number - allResolutions: number[] - hasAudio: boolean - }) { - const { sessionId, videoLive } = options - const videoUUID = videoLive.Video.uuid - const localLTags = lTags(sessionId, videoUUID) - - const liveSession = await this.saveStartingSession(videoLive) - - const user = await UserModel.loadByLiveId(videoLive.id) - LiveQuotaStore.Instance.addNewLive(user.id, sessionId) - - const muxingSession = new MuxingSession({ - context: this.getContext(), - sessionId, - videoLive, - user, - - ...pick(options, [ 'inputLocalUrl', 'inputPublicUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) - }) - - muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) - - muxingSession.on('bad-socket-health', ({ videoUUID }) => { - logger.error( - 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + - ' Stopping session of video %s.', videoUUID, - localLTags - ) - - this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH) - }) - - muxingSession.on('duration-exceeded', ({ videoUUID }) => { - logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) - - this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED) - }) - - muxingSession.on('quota-exceeded', ({ videoUUID }) => { - logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) - - this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED) - }) - - muxingSession.on('transcoding-error', ({ videoUUID }) => { - this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR) - }) - - muxingSession.on('transcoding-end', ({ videoUUID }) => { - this.onMuxingFFmpegEnd(videoUUID, sessionId) - }) - - muxingSession.on('after-cleanup', ({ videoUUID }) => { - this.muxingSessions.delete(sessionId) - - LiveQuotaStore.Instance.removeLive(user.id, sessionId) - - muxingSession.destroy() - - return this.onAfterMuxingCleanup({ videoUUID, liveSession }) - .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) - }) - - this.muxingSessions.set(sessionId, muxingSession) - - muxingSession.runMuxing() - .catch(err => { - logger.error('Cannot run muxing.', { err, ...localLTags }) - this.abortSession(sessionId) - }) - } - - private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { - const videoId = live.videoId - - try { - const video = await VideoModel.loadFull(videoId) - - logger.info('Will publish and federate live %s.', video.url, localLTags) - - video.state = VideoState.PUBLISHED - video.publishedAt = new Date() - await video.save() - - live.Video = video - - await wait(getLiveSegmentTime(live.latencyMode) * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) - - try { - await federateVideoIfNeeded(video, false) - } catch (err) { - logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }) - } - - PeerTubeSocket.Instance.sendVideoLiveNewState(video) - - Hooks.runAction('action:live.video.state.updated', { video }) - } catch (err) { - logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) - } - } - - private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) { - // Session already cleaned up - if (!this.videoSessions.has(videoUUID)) return - - this.videoSessions.delete(videoUUID) - - this.saveEndingSession(videoUUID, null) - .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) - } - - private async onAfterMuxingCleanup (options: { - videoUUID: string - liveSession?: MVideoLiveSession - cleanupNow?: boolean // Default false - }) { - const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options - - logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID)) - - try { - const fullVideo = await VideoModel.loadFull(videoUUID) - if (!fullVideo) return - - const live = await VideoLiveModel.loadByVideoId(fullVideo.id) - - const liveSession = liveSessionArg ?? await VideoLiveSessionModel.findLatestSessionOf(fullVideo.id) - - // On server restart during a live - if (!liveSession.endDate) { - liveSession.endDate = new Date() - await liveSession.save() - } - - JobQueue.Instance.createJobAsync({ - type: 'video-live-ending', - payload: { - videoId: fullVideo.id, - - replayDirectory: live.saveReplay - ? await this.findReplayDirectory(fullVideo) - : undefined, - - liveSessionId: liveSession.id, - streamingPlaylistId: fullVideo.getHLSPlaylist()?.id, - - publishedAt: fullVideo.publishedAt.toISOString() - }, - - delay: cleanupNow - ? 0 - : VIDEO_LIVE.CLEANUP_DELAY - }) - - fullVideo.state = live.permanentLive - ? VideoState.WAITING_FOR_LIVE - : VideoState.LIVE_ENDED - - await fullVideo.save() - - PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) - - await federateVideoIfNeeded(fullVideo, false) - - Hooks.runAction('action:live.video.state.updated', { video: fullVideo }) - } catch (err) { - logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) }) - } - } - - private async handleBrokenLives () { - await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' }) - - const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() - - for (const uuid of videoUUIDs) { - await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true }) - } - } - - private async findReplayDirectory (video: MVideo) { - const directory = getLiveReplayBaseDirectory(video) - const files = await readdir(directory) - - if (files.length === 0) return undefined - - return join(directory, files.sort().reverse()[0]) - } - - private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) { - const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - - const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED - ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio }) - : [] - - if (resolutionsEnabled.length === 0) { - return [ originResolution ] - } - - return resolutionsEnabled - } - - private async saveStartingSession (videoLive: MVideoLiveVideoWithSetting) { - const replaySettings = videoLive.saveReplay - ? new VideoLiveReplaySettingModel({ - privacy: videoLive.ReplaySetting.privacy - }) - : null - - return sequelizeTypescript.transaction(async t => { - if (videoLive.saveReplay) { - await replaySettings.save({ transaction: t }) - } - - return VideoLiveSessionModel.create({ - startDate: new Date(), - liveVideoId: videoLive.videoId, - saveReplay: videoLive.saveReplay, - replaySettingId: videoLive.saveReplay ? replaySettings.id : null, - endingProcessed: false - }, { transaction: t }) - }) - } - - private async saveEndingSession (videoUUID: string, error: LiveVideoError | null) { - const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID) - if (!liveSession) return - - liveSession.endDate = new Date() - liveSession.error = error - - return liveSession.save() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - LiveManager -} diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts deleted file mode 100644 index 44539faaa..000000000 --- a/server/lib/live/live-quota-store.ts +++ /dev/null @@ -1,48 +0,0 @@ -class LiveQuotaStore { - - private static instance: LiveQuotaStore - - private readonly livesPerUser = new Map() - - private constructor () { - } - - addNewLive (userId: number, sessionId: string) { - if (!this.livesPerUser.has(userId)) { - this.livesPerUser.set(userId, []) - } - - const currentUserLive = { sessionId, size: 0 } - const livesOfUser = this.livesPerUser.get(userId) - livesOfUser.push(currentUserLive) - } - - removeLive (userId: number, sessionId: string) { - const newLivesPerUser = this.livesPerUser.get(userId) - .filter(o => o.sessionId !== sessionId) - - this.livesPerUser.set(userId, newLivesPerUser) - } - - addQuotaTo (userId: number, sessionId: string, size: number) { - const lives = this.livesPerUser.get(userId) - const live = lives.find(l => l.sessionId === sessionId) - - live.size += size - } - - getLiveQuotaOf (userId: number) { - const currentLives = this.livesPerUser.get(userId) - if (!currentLives) return 0 - - return currentLives.reduce((sum, obj) => sum + obj.size, 0) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -export { - LiveQuotaStore -} diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts deleted file mode 100644 index 8253c0274..000000000 --- a/server/lib/live/live-segment-sha-store.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { rename, writeJson } from 'fs-extra' -import PQueue from 'p-queue' -import { basename } from 'path' -import { mapToJSON } from '@server/helpers/core-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { MStreamingPlaylistVideo } from '@server/types/models' -import { buildSha256Segment } from '../hls' -import { storeHLSFileFromPath } from '../object-storage' - -const lTags = loggerTagsFactory('live') - -class LiveSegmentShaStore { - - private readonly segmentsSha256 = new Map() - - private readonly videoUUID: string - - private readonly sha256Path: string - private readonly sha256PathTMP: string - - private readonly streamingPlaylist: MStreamingPlaylistVideo - private readonly sendToObjectStorage: boolean - private readonly writeQueue = new PQueue({ concurrency: 1 }) - - constructor (options: { - videoUUID: string - sha256Path: string - streamingPlaylist: MStreamingPlaylistVideo - sendToObjectStorage: boolean - }) { - this.videoUUID = options.videoUUID - - this.sha256Path = options.sha256Path - this.sha256PathTMP = options.sha256Path + '.tmp' - - this.streamingPlaylist = options.streamingPlaylist - this.sendToObjectStorage = options.sendToObjectStorage - } - - async addSegmentSha (segmentPath: string) { - logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID)) - - const shaResult = await buildSha256Segment(segmentPath) - - const segmentName = basename(segmentPath) - this.segmentsSha256.set(segmentName, shaResult) - - try { - await this.writeToDisk() - } catch (err) { - logger.error('Cannot write sha segments to disk.', { err }) - } - } - - async removeSegmentSha (segmentPath: string) { - const segmentName = basename(segmentPath) - - logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) - - if (!this.segmentsSha256.has(segmentName)) { - logger.warn( - 'Unknown segment in live segment hash store for video %s and segment %s.', - this.videoUUID, segmentPath, lTags(this.videoUUID) - ) - return - } - - this.segmentsSha256.delete(segmentName) - - await this.writeToDisk() - } - - private writeToDisk () { - return this.writeQueue.add(async () => { - logger.debug(`Writing segment sha JSON ${this.sha256Path} of ${this.videoUUID} on disk.`, lTags(this.videoUUID)) - - // Atomic write: use rename instead of move that is not atomic - await writeJson(this.sha256PathTMP, mapToJSON(this.segmentsSha256)) - await rename(this.sha256PathTMP, this.sha256Path) - - if (this.sendToObjectStorage) { - const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) - - if (this.streamingPlaylist.segmentsSha256Url !== url) { - this.streamingPlaylist.segmentsSha256Url = url - await this.streamingPlaylist.save() - } - } - }) - } -} - -export { - LiveSegmentShaStore -} diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts deleted file mode 100644 index 3fb3ce1ce..000000000 --- a/server/lib/live/live-utils.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { pathExists, readdir, remove } from 'fs-extra' -import { basename, join } from 'path' -import { logger } from '@server/helpers/logger' -import { VIDEO_LIVE } from '@server/initializers/constants' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' -import { LiveVideoLatencyMode, VideoStorage } from '@shared/models' -import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage' -import { getLiveDirectory } from '../paths' - -function buildConcatenatedName (segmentOrPlaylistPath: string) { - const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) - - return 'concat-' + num[1] + '.ts' -} - -async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - await cleanupTMPLiveFiles(video, streamingPlaylist) - - await streamingPlaylist.destroy() -} - -async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - const hlsDirectory = getLiveDirectory(video) - - // We uploaded files to object storage too, remove them - if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - await removeHLSObjectStorage(streamingPlaylist.withVideo(video)) - } - - await remove(hlsDirectory) - - await streamingPlaylist.destroy() -} - -async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video)) - - await cleanupTMPLiveFilesFromFilesystem(video) -} - -function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) { - if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { - return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY - } - - return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY -} - -export { - cleanupAndDestroyPermanentLive, - cleanupUnsavedNormalLive, - cleanupTMPLiveFiles, - getLiveSegmentTime, - buildConcatenatedName -} - -// --------------------------------------------------------------------------- - -function isTMPLiveFile (name: string) { - return name.endsWith('.ts') || - name.endsWith('.m3u8') || - name.endsWith('.json') || - name.endsWith('.mpd') || - name.endsWith('.m4s') || - name.endsWith('.tmp') -} - -async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) { - const hlsDirectory = getLiveDirectory(video) - - if (!await pathExists(hlsDirectory)) return - - logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory) - - const files = await readdir(hlsDirectory) - - for (const filename of files) { - if (isTMPLiveFile(filename)) { - const p = join(hlsDirectory, filename) - - remove(p) - .catch(err => logger.error('Cannot remove %s.', p, { err })) - } - } -} - -async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { - if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return - - logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid) - - const keys = await listHLSFileKeysOf(streamingPlaylist) - - for (const key of keys) { - if (isTMPLiveFile(key)) { - await removeHLSFileObjectStorageByFullKey(key) - } - } -} diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts deleted file mode 100644 index c4d1b59ec..000000000 --- a/server/lib/live/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './muxing-session' diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts deleted file mode 100644 index 02691b651..000000000 --- a/server/lib/live/shared/muxing-session.ts +++ /dev/null @@ -1,518 +0,0 @@ -import { mapSeries } from 'bluebird' -import { FSWatcher, watch } from 'chokidar' -import { EventEmitter } from 'events' -import { appendFile, ensureDir, readFile, stat } from 'fs-extra' -import PQueue from 'p-queue' -import { basename, join } from 'path' -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants' -import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFileFromPath } from '@server/lib/object-storage' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' -import { VideoStorage, VideoStreamingPlaylistType } from '@shared/models' -import { - generateHLSMasterPlaylistFilename, - generateHlsSha256SegmentsFilename, - getLiveDirectory, - getLiveReplayBaseDirectory -} from '../../paths' -import { isAbleToUploadVideo } from '../../user' -import { LiveQuotaStore } from '../live-quota-store' -import { LiveSegmentShaStore } from '../live-segment-sha-store' -import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils' -import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper' - -import memoizee = require('memoizee') -interface MuxingSessionEvents { - 'live-ready': (options: { videoUUID: string }) => void - - 'bad-socket-health': (options: { videoUUID: string }) => void - 'duration-exceeded': (options: { videoUUID: string }) => void - 'quota-exceeded': (options: { videoUUID: string }) => void - - 'transcoding-end': (options: { videoUUID: string }) => void - 'transcoding-error': (options: { videoUUID: string }) => void - - 'after-cleanup': (options: { videoUUID: string }) => void -} - -declare interface MuxingSession { - on( - event: U, listener: MuxingSessionEvents[U] - ): this - - emit( - event: U, ...args: Parameters - ): boolean -} - -class MuxingSession extends EventEmitter { - - private transcodingWrapper: AbstractTranscodingWrapper - - private readonly context: any - private readonly user: MUserId - private readonly sessionId: string - private readonly videoLive: MVideoLiveVideo - - private readonly inputLocalUrl: string - private readonly inputPublicUrl: string - - private readonly fps: number - private readonly allResolutions: number[] - - private readonly bitrate: number - private readonly ratio: number - - private readonly hasAudio: boolean - - private readonly videoUUID: string - private readonly saveReplay: boolean - - private readonly outDirectory: string - private readonly replayDirectory: string - - private readonly lTags: LoggerTagsFn - - // Path -> Queue - private readonly objectStorageSendQueues = new Map() - - private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} - - private streamingPlaylist: MStreamingPlaylistVideo - private liveSegmentShaStore: LiveSegmentShaStore - - private filesWatcher: FSWatcher - - private masterPlaylistCreated = false - private liveReady = false - - private aborted = false - - private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { - return isAbleToUploadVideo(userId, 1000) - }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) - - private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { - return this.hasClientSocketInBadHealth(sessionId) - }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) - - constructor (options: { - context: any - user: MUserId - sessionId: string - videoLive: MVideoLiveVideo - - inputLocalUrl: string - inputPublicUrl: string - - fps: number - bitrate: number - ratio: number - allResolutions: number[] - hasAudio: boolean - }) { - super() - - this.context = options.context - this.user = options.user - this.sessionId = options.sessionId - this.videoLive = options.videoLive - - this.inputLocalUrl = options.inputLocalUrl - this.inputPublicUrl = options.inputPublicUrl - - this.fps = options.fps - - this.bitrate = options.bitrate - this.ratio = options.ratio - - this.hasAudio = options.hasAudio - - this.allResolutions = options.allResolutions - - this.videoUUID = this.videoLive.Video.uuid - - this.saveReplay = this.videoLive.saveReplay - - this.outDirectory = getLiveDirectory(this.videoLive.Video) - this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) - - this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) - } - - async runMuxing () { - this.streamingPlaylist = await this.createLivePlaylist() - - this.createLiveShaStore() - this.createFiles() - - await this.prepareDirectories() - - this.transcodingWrapper = this.buildTranscodingWrapper() - - this.transcodingWrapper.on('end', () => this.onTranscodedEnded()) - this.transcodingWrapper.on('error', () => this.onTranscodingError()) - - await this.transcodingWrapper.run() - - this.filesWatcher = watch(this.outDirectory, { depth: 0 }) - - this.watchMasterFile() - this.watchTSFiles() - } - - abort () { - if (!this.transcodingWrapper) return - - this.aborted = true - this.transcodingWrapper.abort() - } - - destroy () { - this.removeAllListeners() - this.isAbleToUploadVideoWithCache.clear() - this.hasClientSocketInBadHealthWithCache.clear() - } - - private watchMasterFile () { - this.filesWatcher.on('add', async path => { - if (path !== join(this.outDirectory, this.streamingPlaylist.playlistFilename)) return - if (this.masterPlaylistCreated === true) return - - try { - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - const masterContent = await readFile(path, 'utf-8') - logger.debug('Uploading live master playlist on object storage for %s', this.videoUUID, { masterContent, ...this.lTags() }) - - const url = await storeHLSFileFromContent(this.streamingPlaylist, this.streamingPlaylist.playlistFilename, masterContent) - - this.streamingPlaylist.playlistUrl = url - } - - this.streamingPlaylist.assignP2PMediaLoaderInfoHashes(this.videoLive.Video, this.allResolutions) - - await this.streamingPlaylist.save() - } catch (err) { - logger.error('Cannot update streaming playlist.', { err, ...this.lTags() }) - } - - this.masterPlaylistCreated = true - - logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags()) - }) - } - - private watchTSFiles () { - const startStreamDateTime = new Date().getTime() - - const addHandler = async (segmentPath: string) => { - if (segmentPath.endsWith('.ts') !== true) return - - logger.debug('Live add handler of TS file %s.', segmentPath, this.lTags()) - - const playlistId = this.getPlaylistIdFromTS(segmentPath) - - const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] - this.processSegments(segmentsToProcess) - - this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] - - if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { - this.emit('bad-socket-health', { videoUUID: this.videoUUID }) - return - } - - // Duration constraint check - if (this.isDurationConstraintValid(startStreamDateTime) !== true) { - this.emit('duration-exceeded', { videoUUID: this.videoUUID }) - return - } - - // Check user quota if the user enabled replay saving - if (await this.isQuotaExceeded(segmentPath) === true) { - this.emit('quota-exceeded', { videoUUID: this.videoUUID }) - } - } - - const deleteHandler = async (segmentPath: string) => { - if (segmentPath.endsWith('.ts') !== true) return - - logger.debug('Live delete handler of TS file %s.', segmentPath, this.lTags()) - - try { - await this.liveSegmentShaStore.removeSegmentSha(segmentPath) - } catch (err) { - logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) - } - - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - try { - await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath) - } catch (err) { - logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) - } - } - } - - this.filesWatcher.on('add', p => addHandler(p)) - this.filesWatcher.on('unlink', p => deleteHandler(p)) - } - - private async isQuotaExceeded (segmentPath: string) { - if (this.saveReplay !== true) return false - if (this.aborted) return false - - try { - const segmentStat = await stat(segmentPath) - - LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.sessionId, segmentStat.size) - - const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) - - return canUpload !== true - } catch (err) { - logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags() }) - } - } - - private createFiles () { - for (let i = 0; i < this.allResolutions.length; i++) { - const resolution = this.allResolutions[i] - - const file = new VideoFileModel({ - resolution, - size: -1, - extname: '.ts', - infoHash: null, - fps: this.fps, - storage: this.streamingPlaylist.storage, - videoStreamingPlaylistId: this.streamingPlaylist.id - }) - - VideoFileModel.customUpsert(file, 'streaming-playlist', null) - .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags() })) - } - } - - private async prepareDirectories () { - await ensureDir(this.outDirectory) - - if (this.videoLive.saveReplay === true) { - await ensureDir(this.replayDirectory) - } - } - - private isDurationConstraintValid (streamingStartTime: number) { - const maxDuration = CONFIG.LIVE.MAX_DURATION - // No limit - if (maxDuration < 0) return true - - const now = new Date().getTime() - const max = streamingStartTime + maxDuration - - return now <= max - } - - private processSegments (segmentPaths: string[]) { - mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment)) - .catch(err => { - if (this.aborted) return - - logger.error('Cannot process segments', { err, ...this.lTags() }) - }) - } - - private async processSegment (segmentPath: string) { - // Add sha hash of previous segments, because ffmpeg should have finished generating them - await this.liveSegmentShaStore.addSegmentSha(segmentPath) - - if (this.saveReplay) { - await this.addSegmentToReplay(segmentPath) - } - - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - try { - await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) - - await this.processM3U8ToObjectStorage(segmentPath) - } catch (err) { - logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() }) - } - } - - // Master playlist and segment JSON file are created, live is ready - if (this.masterPlaylistCreated && !this.liveReady) { - this.liveReady = true - - this.emit('live-ready', { videoUUID: this.videoUUID }) - } - } - - private async processM3U8ToObjectStorage (segmentPath: string) { - const m3u8Path = join(this.outDirectory, this.getPlaylistNameFromTS(segmentPath)) - - logger.debug('Process M3U8 file %s.', m3u8Path, this.lTags()) - - const segmentName = basename(segmentPath) - - const playlistContent = await readFile(m3u8Path, 'utf-8') - // Remove new chunk references, that will be processed later - const filteredPlaylistContent = playlistContent.substring(0, playlistContent.lastIndexOf(segmentName) + segmentName.length) + '\n' - - try { - if (!this.objectStorageSendQueues.has(m3u8Path)) { - this.objectStorageSendQueues.set(m3u8Path, new PQueue({ concurrency: 1 })) - } - - const queue = this.objectStorageSendQueues.get(m3u8Path) - await queue.add(() => storeHLSFileFromContent(this.streamingPlaylist, m3u8Path, filteredPlaylistContent)) - } catch (err) { - logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() }) - } - } - - private onTranscodingError () { - this.emit('transcoding-error', ({ videoUUID: this.videoUUID })) - } - - private onTranscodedEnded () { - this.emit('transcoding-end', ({ videoUUID: this.videoUUID })) - - logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputLocalUrl, this.lTags()) - - setTimeout(() => { - // Wait latest segments generation, and close watchers - - const promise = this.filesWatcher?.close() || Promise.resolve() - promise - .then(() => { - // Process remaining segments hash - for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { - this.processSegments(this.segmentsToProcessPerPlaylist[key]) - } - }) - .catch(err => { - logger.error( - 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, - { err, ...this.lTags() } - ) - }) - - this.emit('after-cleanup', { videoUUID: this.videoUUID }) - }, 1000) - } - - private hasClientSocketInBadHealth (sessionId: string) { - const rtmpSession = this.context.sessions.get(sessionId) - - if (!rtmpSession) { - logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags()) - return - } - - for (const playerSessionId of rtmpSession.players) { - const playerSession = this.context.sessions.get(playerSessionId) - - if (!playerSession) { - logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags()) - continue - } - - if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { - return true - } - } - - return false - } - - private async addSegmentToReplay (segmentPath: string) { - const segmentName = basename(segmentPath) - const dest = join(this.replayDirectory, buildConcatenatedName(segmentName)) - - try { - const data = await readFile(segmentPath) - - await appendFile(dest, data) - } catch (err) { - logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags() }) - } - } - - private async createLivePlaylist (): Promise { - const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(this.videoLive.Video) - - playlist.playlistFilename = generateHLSMasterPlaylistFilename(true) - playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true) - - playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - playlist.type = VideoStreamingPlaylistType.HLS - - playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED - ? VideoStorage.OBJECT_STORAGE - : VideoStorage.FILE_SYSTEM - - return playlist.save() - } - - private createLiveShaStore () { - this.liveSegmentShaStore = new LiveSegmentShaStore({ - videoUUID: this.videoLive.Video.uuid, - sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename), - streamingPlaylist: this.streamingPlaylist, - sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED - }) - } - - private buildTranscodingWrapper () { - const options = { - streamingPlaylist: this.streamingPlaylist, - videoLive: this.videoLive, - - lTags: this.lTags, - - sessionId: this.sessionId, - inputLocalUrl: this.inputLocalUrl, - inputPublicUrl: this.inputPublicUrl, - - toTranscode: this.allResolutions.map(resolution => ({ - resolution, - fps: computeOutputFPS({ inputFPS: this.fps, resolution }) - })), - - fps: this.fps, - bitrate: this.bitrate, - ratio: this.ratio, - hasAudio: this.hasAudio, - - segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, - segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode), - - outDirectory: this.outDirectory - } - - return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED - ? new RemoteTranscodingWrapper(options) - : new FFmpegTranscodingWrapper(options) - } - - private getPlaylistIdFromTS (segmentPath: string) { - const playlistIdMatcher = /^([\d+])-/ - - return basename(segmentPath).match(playlistIdMatcher)[1] - } - - private getPlaylistNameFromTS (segmentPath: string) { - return `${this.getPlaylistIdFromTS(segmentPath)}.m3u8` - } -} - -// --------------------------------------------------------------------------- - -export { - MuxingSession -} diff --git a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts deleted file mode 100644 index 95168745d..000000000 --- a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts +++ /dev/null @@ -1,110 +0,0 @@ -import EventEmitter from 'events' -import { LoggerTagsFn } from '@server/helpers/logger' -import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models' -import { LiveVideoError } from '@shared/models' - -interface TranscodingWrapperEvents { - 'end': () => void - - 'error': (options: { err: Error }) => void -} - -declare interface AbstractTranscodingWrapper { - on( - event: U, listener: TranscodingWrapperEvents[U] - ): this - - emit( - event: U, ...args: Parameters - ): boolean -} - -interface AbstractTranscodingWrapperOptions { - streamingPlaylist: MStreamingPlaylistVideo - videoLive: MVideoLiveVideo - - lTags: LoggerTagsFn - - sessionId: string - inputLocalUrl: string - inputPublicUrl: string - - fps: number - toTranscode: { - resolution: number - fps: number - }[] - - bitrate: number - ratio: number - hasAudio: boolean - - segmentListSize: number - segmentDuration: number - - outDirectory: string -} - -abstract class AbstractTranscodingWrapper extends EventEmitter { - protected readonly videoLive: MVideoLiveVideo - - protected readonly toTranscode: { - resolution: number - fps: number - }[] - - protected readonly sessionId: string - protected readonly inputLocalUrl: string - protected readonly inputPublicUrl: string - - protected readonly fps: number - protected readonly bitrate: number - protected readonly ratio: number - protected readonly hasAudio: boolean - - protected readonly segmentListSize: number - protected readonly segmentDuration: number - - protected readonly videoUUID: string - - protected readonly outDirectory: string - - protected readonly lTags: LoggerTagsFn - - protected readonly streamingPlaylist: MStreamingPlaylistVideo - - constructor (options: AbstractTranscodingWrapperOptions) { - super() - - this.lTags = options.lTags - - this.videoLive = options.videoLive - this.videoUUID = options.videoLive.Video.uuid - this.streamingPlaylist = options.streamingPlaylist - - this.sessionId = options.sessionId - this.inputLocalUrl = options.inputLocalUrl - this.inputPublicUrl = options.inputPublicUrl - - this.fps = options.fps - this.toTranscode = options.toTranscode - - this.bitrate = options.bitrate - this.ratio = options.ratio - this.hasAudio = options.hasAudio - - this.segmentListSize = options.segmentListSize - this.segmentDuration = options.segmentDuration - - this.outDirectory = options.outDirectory - } - - abstract run (): Promise - - abstract abort (error?: LiveVideoError): void -} - -export { - AbstractTranscodingWrapper, - AbstractTranscodingWrapperOptions -} diff --git a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts deleted file mode 100644 index c6ee8ebf1..000000000 --- a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { FfmpegCommand } from 'fluent-ffmpeg' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { VIDEO_LIVE } from '@server/initializers/constants' -import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' -import { FFmpegLive } from '@shared/ffmpeg' -import { getLiveSegmentTime } from '../../live-utils' -import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' - -export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper { - private ffmpegCommand: FfmpegCommand - - private aborted = false - private errored = false - private ended = false - - async run () { - this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED - ? await this.buildFFmpegLive().getLiveTranscodingCommand({ - inputUrl: this.inputLocalUrl, - - outPath: this.outDirectory, - masterPlaylistName: this.streamingPlaylist.playlistFilename, - - segmentListSize: this.segmentListSize, - segmentDuration: this.segmentDuration, - - toTranscode: this.toTranscode, - - bitrate: this.bitrate, - ratio: this.ratio, - - hasAudio: this.hasAudio - }) - : this.buildFFmpegLive().getLiveMuxingCommand({ - inputUrl: this.inputLocalUrl, - outPath: this.outDirectory, - - masterPlaylistName: this.streamingPlaylist.playlistFilename, - - segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, - segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode) - }) - - logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags()) - - let ffmpegShellCommand: string - this.ffmpegCommand.on('start', cmdline => { - ffmpegShellCommand = cmdline - - logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) - }) - - this.ffmpegCommand.on('error', (err, stdout, stderr) => { - this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) - }) - - this.ffmpegCommand.on('end', () => { - this.onFFmpegEnded() - }) - - this.ffmpegCommand.run() - } - - abort () { - if (this.ended || this.errored || this.aborted) return - - logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags()) - - this.ffmpegCommand.kill('SIGINT') - - this.aborted = true - this.emit('end') - } - - private onFFmpegError (options: { - err: any - stdout: string - stderr: string - ffmpegShellCommand: string - }) { - const { err, stdout, stderr, ffmpegShellCommand } = options - - // Don't care that we killed the ffmpeg process - if (err?.message?.includes('Exiting normally')) return - if (this.ended || this.errored || this.aborted) return - - logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) - - this.errored = true - this.emit('error', { err }) - } - - private onFFmpegEnded () { - if (this.ended || this.errored || this.aborted) return - - logger.debug('Live ffmpeg transcoding ended for ' + this.videoUUID, this.lTags()) - - this.ended = true - this.emit('end') - } - - private buildFFmpegLive () { - return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) - } -} diff --git a/server/lib/live/shared/transcoding-wrapper/index.ts b/server/lib/live/shared/transcoding-wrapper/index.ts deleted file mode 100644 index ae28fa1ca..000000000 --- a/server/lib/live/shared/transcoding-wrapper/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './abstract-transcoding-wrapper' -export * from './ffmpeg-transcoding-wrapper' -export * from './remote-transcoding-wrapper' diff --git a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts deleted file mode 100644 index 2aeeb31fb..000000000 --- a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners' -import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' - -export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper { - async run () { - await new LiveRTMPHLSTranscodingJobHandler().create({ - rtmpUrl: this.inputPublicUrl, - sessionId: this.sessionId, - toTranscode: this.toTranscode, - video: this.videoLive.Video, - outputDirectory: this.outDirectory, - playlist: this.streamingPlaylist, - segmentListSize: this.segmentListSize, - segmentDuration: this.segmentDuration - }) - } - - abort () { - this.emit('end') - } -} diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts deleted file mode 100644 index 611e6d0af..000000000 --- a/server/lib/local-actor.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { Transaction } from 'sequelize/types' -import { ActorModel } from '@server/models/actor/actor' -import { getLowercaseExtension } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { ActivityPubActorType, ActorImageType } from '@shared/models' -import { retryTransactionWrapper } from '../helpers/database-utils' -import { CONFIG } from '../initializers/config' -import { ACTOR_IMAGES_SIZE, WEBSERVER } from '../initializers/constants' -import { sequelizeTypescript } from '../initializers/database' -import { MAccountDefault, MActor, MChannelDefault } from '../types/models' -import { deleteActorImages, updateActorImages } from './activitypub/actors' -import { sendUpdateActor } from './activitypub/send' -import { processImageFromWorker } from './worker/parent-process' - -export function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { - return new ActorModel({ - type, - url, - preferredUsername, - publicKey: null, - privateKey: null, - followersCount: 0, - followingCount: 0, - inboxUrl: url + '/inbox', - outboxUrl: url + '/outbox', - sharedInboxUrl: WEBSERVER.URL + '/inbox', - followersUrl: url + '/followers', - followingUrl: url + '/following' - }) as MActor -} - -export async function updateLocalActorImageFiles ( - accountOrChannel: MAccountDefault | MChannelDefault, - imagePhysicalFile: Express.Multer.File, - type: ActorImageType -) { - const processImageSize = async (imageSize: { width: number, height: number }) => { - const extension = getLowercaseExtension(imagePhysicalFile.filename) - - const imageName = buildUUID() + extension - const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName) - await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) - - return { - imageName, - imageSize - } - } - - const processedImages = await Promise.all(ACTOR_IMAGES_SIZE[type].map(processImageSize)) - await remove(imagePhysicalFile.path) - - return retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { - const actorImagesInfo = processedImages.map(({ imageName, imageSize }) => ({ - name: imageName, - fileUrl: null, - height: imageSize.height, - width: imageSize.width, - onDisk: true - })) - - const updatedActor = await updateActorImages(accountOrChannel.Actor, type, actorImagesInfo, t) - await updatedActor.save({ transaction: t }) - - await sendUpdateActor(accountOrChannel, t) - - return type === ActorImageType.AVATAR - ? updatedActor.Avatars - : updatedActor.Banners - })) -} - -export async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) { - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t) - await updatedActor.save({ transaction: t }) - - await sendUpdateActor(accountOrChannel, t) - - return updatedActor.Avatars - }) - }) -} - -// --------------------------------------------------------------------------- - -export async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) { - let actor = await ActorModel.loadLocalByName(baseActorName, transaction) - if (!actor) return baseActorName - - for (let i = 1; i < 30; i++) { - const name = `${baseActorName}-${i}` - - actor = await ActorModel.loadLocalByName(name, transaction) - if (!actor) return name - } - - throw new Error('Cannot find available actor local name (too much iterations).') -} diff --git a/server/lib/model-loaders/actor.ts b/server/lib/model-loaders/actor.ts deleted file mode 100644 index 1355d8ee2..000000000 --- a/server/lib/model-loaders/actor.ts +++ /dev/null @@ -1,17 +0,0 @@ - -import { ActorModel } from '../../models/actor/actor' -import { MActorAccountChannelId, MActorFull } from '../../types/models' - -type ActorLoadByUrlType = 'all' | 'association-ids' - -function loadActorByUrl (url: string, fetchType: ActorLoadByUrlType): Promise { - if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url) - - if (fetchType === 'association-ids') return ActorModel.loadByUrl(url) -} - -export { - ActorLoadByUrlType, - - loadActorByUrl -} diff --git a/server/lib/model-loaders/index.ts b/server/lib/model-loaders/index.ts deleted file mode 100644 index 9e5152cb2..000000000 --- a/server/lib/model-loaders/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './actor' -export * from './video' diff --git a/server/lib/model-loaders/video.ts b/server/lib/model-loaders/video.ts deleted file mode 100644 index 91057d405..000000000 --- a/server/lib/model-loaders/video.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { - MVideoAccountLightBlacklistAllFiles, - MVideoFormattableDetails, - MVideoFullLight, - MVideoId, - MVideoImmutable, - MVideoThumbnail -} from '@server/types/models' - -type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes' - -function loadVideo (id: number | string, fetchType: 'for-api', userId?: number): Promise -function loadVideo (id: number | string, fetchType: 'all', userId?: number): Promise -function loadVideo (id: number | string, fetchType: 'only-immutable-attributes'): Promise -function loadVideo (id: number | string, fetchType: 'only-video', userId?: number): Promise -function loadVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Promise -function loadVideo ( - id: number | string, - fetchType: VideoLoadType, - userId?: number -): Promise -function loadVideo ( - id: number | string, - fetchType: VideoLoadType, - userId?: number -): Promise { - - if (fetchType === 'for-api') return VideoModel.loadForGetAPI({ id, userId }) - - if (fetchType === 'all') return VideoModel.loadFull(id, undefined, userId) - - if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id) - - if (fetchType === 'only-video') return VideoModel.load(id) - - if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) -} - -type VideoLoadByUrlType = 'all' | 'only-video' | 'only-immutable-attributes' - -function loadVideoByUrl (url: string, fetchType: 'all'): Promise -function loadVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Promise -function loadVideoByUrl (url: string, fetchType: 'only-video'): Promise -function loadVideoByUrl ( - url: string, - fetchType: VideoLoadByUrlType -): Promise -function loadVideoByUrl ( - url: string, - fetchType: VideoLoadByUrlType -): Promise { - if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) - - if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url) - - if (fetchType === 'only-video') return VideoModel.loadByUrl(url) -} - -export { - VideoLoadType, - VideoLoadByUrlType, - - loadVideo, - loadVideoByUrl -} diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts deleted file mode 100644 index db8284872..000000000 --- a/server/lib/moderation.ts +++ /dev/null @@ -1,258 +0,0 @@ -import express, { VideoUploadFile } from 'express' -import { PathLike } from 'fs-extra' -import { Transaction } from 'sequelize/types' -import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { AbuseModel } from '@server/models/abuse/abuse' -import { VideoAbuseModel } from '@server/models/abuse/video-abuse' -import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse' -import { VideoFileModel } from '@server/models/video/video-file' -import { FilteredModelAttributes } from '@server/types' -import { - MAbuseFull, - MAccountDefault, - MAccountLight, - MComment, - MCommentAbuseAccountVideo, - MCommentOwnerVideo, - MUser, - MVideoAbuseVideoFull, - MVideoAccountLightBlacklistAllFiles -} from '@server/types/models' -import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' -import { VideoCommentCreate } from '../../shared/models/videos/comment' -import { UserModel } from '../models/user/user' -import { VideoModel } from '../models/video/video' -import { VideoCommentModel } from '../models/video/video-comment' -import { sendAbuse } from './activitypub/send/send-flag' -import { Notifier } from './notifier' - -export type AcceptResult = { - accepted: boolean - errorMessage?: string -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isLocalVideoFileAccepted (object: { - videoBody: VideoCreate - videoFile: VideoUploadFile - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isLocalLiveVideoAccepted (object: { - liveVideoBody: LiveVideoCreate - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isLocalVideoThreadAccepted (_object: { - req: express.Request - commentBody: VideoCommentCreate - video: VideoModel - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// Stub function that can be filtered by plugins -function isLocalVideoCommentReplyAccepted (_object: { - req: express.Request - commentBody: VideoCommentCreate - parentComment: VideoCommentModel - video: VideoModel - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isRemoteVideoCommentAccepted (_object: { - comment: MComment -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isPreImportVideoAccepted (object: { - videoImportBody: VideoImportCreate - user: MUser -}): AcceptResult { - return { accepted: true } -} - -// Stub function that can be filtered by plugins -function isPostImportVideoAccepted (object: { - videoFilePath: PathLike - videoFile: VideoFileModel - user: MUser -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -async function createVideoAbuse (options: { - baseAbuse: FilteredModelAttributes - videoInstance: MVideoAccountLightBlacklistAllFiles - startAt: number - endAt: number - transaction: Transaction - reporterAccount: MAccountDefault - skipNotification: boolean -}) { - const { baseAbuse, videoInstance, startAt, endAt, transaction, reporterAccount, skipNotification } = options - - const associateFun = async (abuseInstance: MAbuseFull) => { - const videoAbuseInstance: MVideoAbuseVideoFull = await VideoAbuseModel.create({ - abuseId: abuseInstance.id, - videoId: videoInstance.id, - startAt, - endAt - }, { transaction }) - - videoAbuseInstance.Video = videoInstance - abuseInstance.VideoAbuse = videoAbuseInstance - - return { isOwned: videoInstance.isOwned() } - } - - return createAbuse({ - base: baseAbuse, - reporterAccount, - flaggedAccount: videoInstance.VideoChannel.Account, - transaction, - skipNotification, - associateFun - }) -} - -function createVideoCommentAbuse (options: { - baseAbuse: FilteredModelAttributes - commentInstance: MCommentOwnerVideo - transaction: Transaction - reporterAccount: MAccountDefault - skipNotification: boolean -}) { - const { baseAbuse, commentInstance, transaction, reporterAccount, skipNotification } = options - - const associateFun = async (abuseInstance: MAbuseFull) => { - const commentAbuseInstance: MCommentAbuseAccountVideo = await VideoCommentAbuseModel.create({ - abuseId: abuseInstance.id, - videoCommentId: commentInstance.id - }, { transaction }) - - commentAbuseInstance.VideoComment = commentInstance - abuseInstance.VideoCommentAbuse = commentAbuseInstance - - return { isOwned: commentInstance.isOwned() } - } - - return createAbuse({ - base: baseAbuse, - reporterAccount, - flaggedAccount: commentInstance.Account, - transaction, - skipNotification, - associateFun - }) -} - -function createAccountAbuse (options: { - baseAbuse: FilteredModelAttributes - accountInstance: MAccountDefault - transaction: Transaction - reporterAccount: MAccountDefault - skipNotification: boolean -}) { - const { baseAbuse, accountInstance, transaction, reporterAccount, skipNotification } = options - - const associateFun = () => { - return Promise.resolve({ isOwned: accountInstance.isOwned() }) - } - - return createAbuse({ - base: baseAbuse, - reporterAccount, - flaggedAccount: accountInstance, - transaction, - skipNotification, - associateFun - }) -} - -// --------------------------------------------------------------------------- - -export { - isLocalLiveVideoAccepted, - - isLocalVideoFileAccepted, - isLocalVideoThreadAccepted, - isRemoteVideoCommentAccepted, - isLocalVideoCommentReplyAccepted, - isPreImportVideoAccepted, - isPostImportVideoAccepted, - - createAbuse, - createVideoAbuse, - createVideoCommentAbuse, - createAccountAbuse -} - -// --------------------------------------------------------------------------- - -async function createAbuse (options: { - base: FilteredModelAttributes - reporterAccount: MAccountDefault - flaggedAccount: MAccountLight - associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean }> - skipNotification: boolean - transaction: Transaction -}) { - const { base, reporterAccount, flaggedAccount, associateFun, transaction, skipNotification } = options - const auditLogger = auditLoggerFactory('abuse') - - const abuseAttributes = Object.assign({}, base, { flaggedAccountId: flaggedAccount.id }) - const abuseInstance: MAbuseFull = await AbuseModel.create(abuseAttributes, { transaction }) - - abuseInstance.ReporterAccount = reporterAccount - abuseInstance.FlaggedAccount = flaggedAccount - - const { isOwned } = await associateFun(abuseInstance) - - if (isOwned === false) { - sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction) - } - - const abuseJSON = abuseInstance.toFormattedAdminJSON() - auditLogger.create(reporterAccount.Actor.getIdentifier(), new AbuseAuditView(abuseJSON)) - - if (!skipNotification) { - afterCommitIfTransaction(transaction, () => { - Notifier.Instance.notifyOnNewAbuse({ - abuse: abuseJSON, - abuseInstance, - reporter: reporterAccount.Actor.getIdentifier() - }) - }) - } - - logger.info('Abuse report %d created.', abuseInstance.id) - - return abuseJSON -} diff --git a/server/lib/notifier/index.ts b/server/lib/notifier/index.ts deleted file mode 100644 index 5bc2f5f50..000000000 --- a/server/lib/notifier/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './notifier' diff --git a/server/lib/notifier/notifier.ts b/server/lib/notifier/notifier.ts deleted file mode 100644 index 920c55df0..000000000 --- a/server/lib/notifier/notifier.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { MRegistration, MUser, MUserDefault } from '@server/types/models/user' -import { MVideoBlacklistLightVideo, MVideoBlacklistVideo } from '@server/types/models/video/video-blacklist' -import { UserNotificationSettingValue } from '../../../shared/models/users' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { MAbuseFull, MAbuseMessage, MActorFollowFull, MApplication, MPlugin } from '../../types/models' -import { MCommentOwnerVideo, MVideoAccountLight, MVideoFullLight } from '../../types/models/video' -import { JobQueue } from '../job-queue' -import { PeerTubeSocket } from '../peertube-socket' -import { Hooks } from '../plugins/hooks' -import { - AbstractNotification, - AbuseStateChangeForReporter, - AutoFollowForInstance, - CommentMention, - DirectRegistrationForModerators, - FollowForInstance, - FollowForUser, - ImportFinishedForOwner, - ImportFinishedForOwnerPayload, - NewAbuseForModerators, - NewAbuseMessageForModerators, - NewAbuseMessageForReporter, - NewAbusePayload, - NewAutoBlacklistForModerators, - NewBlacklistForOwner, - NewCommentForVideoOwner, - NewPeerTubeVersionForAdmins, - NewPluginVersionForAdmins, - NewVideoForSubscribers, - OwnedPublicationAfterAutoUnblacklist, - OwnedPublicationAfterScheduleUpdate, - OwnedPublicationAfterTranscoding, - RegistrationRequestForModerators, - StudioEditionFinishedForOwner, - UnblacklistForOwner -} from './shared' - -class Notifier { - - private readonly notificationModels = { - newVideo: [ NewVideoForSubscribers ], - publicationAfterTranscoding: [ OwnedPublicationAfterTranscoding ], - publicationAfterScheduleUpdate: [ OwnedPublicationAfterScheduleUpdate ], - publicationAfterAutoUnblacklist: [ OwnedPublicationAfterAutoUnblacklist ], - newComment: [ CommentMention, NewCommentForVideoOwner ], - newAbuse: [ NewAbuseForModerators ], - newBlacklist: [ NewBlacklistForOwner ], - unblacklist: [ UnblacklistForOwner ], - importFinished: [ ImportFinishedForOwner ], - directRegistration: [ DirectRegistrationForModerators ], - registrationRequest: [ RegistrationRequestForModerators ], - userFollow: [ FollowForUser ], - instanceFollow: [ FollowForInstance ], - autoInstanceFollow: [ AutoFollowForInstance ], - newAutoBlacklist: [ NewAutoBlacklistForModerators ], - abuseStateChange: [ AbuseStateChangeForReporter ], - newAbuseMessage: [ NewAbuseMessageForReporter, NewAbuseMessageForModerators ], - newPeertubeVersion: [ NewPeerTubeVersionForAdmins ], - newPluginVersion: [ NewPluginVersionForAdmins ], - videoStudioEditionFinished: [ StudioEditionFinishedForOwner ] - } - - private static instance: Notifier - - private constructor () { - } - - notifyOnNewVideoIfNeeded (video: MVideoAccountLight): void { - const models = this.notificationModels.newVideo - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) - } - - notifyOnVideoPublishedAfterTranscoding (video: MVideoFullLight): void { - const models = this.notificationModels.publicationAfterTranscoding - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify owner that its video %s has been published after transcoding.', video.url, { err })) - } - - notifyOnVideoPublishedAfterScheduledUpdate (video: MVideoFullLight): void { - const models = this.notificationModels.publicationAfterScheduleUpdate - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify owner that its video %s has been published after scheduled update.', video.url, { err })) - } - - notifyOnVideoPublishedAfterRemovedFromAutoBlacklist (video: MVideoFullLight): void { - const models = this.notificationModels.publicationAfterAutoUnblacklist - - this.sendNotifications(models, video) - .catch(err => { - logger.error('Cannot notify owner that its video %s has been published after removed from auto-blacklist.', video.url, { err }) - }) - } - - notifyOnNewComment (comment: MCommentOwnerVideo): void { - const models = this.notificationModels.newComment - - this.sendNotifications(models, comment) - .catch(err => logger.error('Cannot notify of new comment.', comment.url, { err })) - } - - notifyOnNewAbuse (payload: NewAbusePayload): void { - const models = this.notificationModels.newAbuse - - this.sendNotifications(models, payload) - .catch(err => logger.error('Cannot notify of new abuse %d.', payload.abuseInstance.id, { err })) - } - - notifyOnVideoAutoBlacklist (videoBlacklist: MVideoBlacklistLightVideo): void { - const models = this.notificationModels.newAutoBlacklist - - this.sendNotifications(models, videoBlacklist) - .catch(err => logger.error('Cannot notify of auto-blacklist of video %s.', videoBlacklist.Video.url, { err })) - } - - notifyOnVideoBlacklist (videoBlacklist: MVideoBlacklistVideo): void { - const models = this.notificationModels.newBlacklist - - this.sendNotifications(models, videoBlacklist) - .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) - } - - notifyOnVideoUnblacklist (video: MVideoFullLight): void { - const models = this.notificationModels.unblacklist - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify video owner of unblacklist of %s.', video.url, { err })) - } - - notifyOnFinishedVideoImport (payload: ImportFinishedForOwnerPayload): void { - const models = this.notificationModels.importFinished - - this.sendNotifications(models, payload) - .catch(err => { - logger.error('Cannot notify owner that its video import %s is finished.', payload.videoImport.getTargetIdentifier(), { err }) - }) - } - - notifyOnNewDirectRegistration (user: MUserDefault): void { - const models = this.notificationModels.directRegistration - - this.sendNotifications(models, user) - .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) - } - - notifyOnNewRegistrationRequest (registration: MRegistration): void { - const models = this.notificationModels.registrationRequest - - this.sendNotifications(models, registration) - .catch(err => logger.error('Cannot notify moderators of new registration request (%s).', registration.username, { err })) - } - - notifyOfNewUserFollow (actorFollow: MActorFollowFull): void { - const models = this.notificationModels.userFollow - - this.sendNotifications(models, actorFollow) - .catch(err => { - logger.error( - 'Cannot notify owner of channel %s of a new follow by %s.', - actorFollow.ActorFollowing.VideoChannel.getDisplayName(), - actorFollow.ActorFollower.Account.getDisplayName(), - { err } - ) - }) - } - - notifyOfNewInstanceFollow (actorFollow: MActorFollowFull): void { - const models = this.notificationModels.instanceFollow - - this.sendNotifications(models, actorFollow) - .catch(err => logger.error('Cannot notify administrators of new follower %s.', actorFollow.ActorFollower.url, { err })) - } - - notifyOfAutoInstanceFollowing (actorFollow: MActorFollowFull): void { - const models = this.notificationModels.autoInstanceFollow - - this.sendNotifications(models, actorFollow) - .catch(err => logger.error('Cannot notify administrators of auto instance following %s.', actorFollow.ActorFollowing.url, { err })) - } - - notifyOnAbuseStateChange (abuse: MAbuseFull): void { - const models = this.notificationModels.abuseStateChange - - this.sendNotifications(models, abuse) - .catch(err => logger.error('Cannot notify of abuse %d state change.', abuse.id, { err })) - } - - notifyOnAbuseMessage (abuse: MAbuseFull, message: MAbuseMessage): void { - const models = this.notificationModels.newAbuseMessage - - this.sendNotifications(models, { abuse, message }) - .catch(err => logger.error('Cannot notify on new abuse %d message.', abuse.id, { err })) - } - - notifyOfNewPeerTubeVersion (application: MApplication, latestVersion: string) { - const models = this.notificationModels.newPeertubeVersion - - this.sendNotifications(models, { application, latestVersion }) - .catch(err => logger.error('Cannot notify on new PeerTubeb version %s.', latestVersion, { err })) - } - - notifyOfNewPluginVersion (plugin: MPlugin) { - const models = this.notificationModels.newPluginVersion - - this.sendNotifications(models, plugin) - .catch(err => logger.error('Cannot notify on new plugin version %s.', plugin.name, { err })) - } - - notifyOfFinishedVideoStudioEdition (video: MVideoFullLight) { - const models = this.notificationModels.videoStudioEditionFinished - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify on finished studio edition %s.', video.url, { err })) - } - - private async notify (object: AbstractNotification) { - await object.prepare() - - const users = object.getTargetUsers() - - if (users.length === 0) return - if (await object.isDisabled()) return - - object.log() - - const toEmails: string[] = [] - - for (const user of users) { - const setting = object.getSetting(user) - - const webNotificationEnabled = this.isWebNotificationEnabled(setting) - const emailNotificationEnabled = this.isEmailEnabled(user, setting) - const notification = object.createNotification(user) - - if (webNotificationEnabled) { - await notification.save() - - PeerTubeSocket.Instance.sendNotification(user.id, notification) - } - - if (emailNotificationEnabled) { - toEmails.push(user.email) - } - - Hooks.runAction('action:notifier.notification.created', { webNotificationEnabled, emailNotificationEnabled, user, notification }) - } - - for (const to of toEmails) { - const payload = await object.createEmail(to) - JobQueue.Instance.createJobAsync({ type: 'email', payload }) - } - } - - private isEmailEnabled (user: MUser, value: UserNotificationSettingValue) { - if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified === false) return false - - return value & UserNotificationSettingValue.EMAIL - } - - private isWebNotificationEnabled (value: UserNotificationSettingValue) { - return value & UserNotificationSettingValue.WEB - } - - private async sendNotifications (models: (new (payload: T) => AbstractNotification)[], payload: T) { - for (const model of models) { - // eslint-disable-next-line new-cap - await this.notify(new model(payload)) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - Notifier -} diff --git a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts deleted file mode 100644 index 1dc1ccfc2..000000000 --- a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { WEBSERVER } from '@server/initializers/constants' -import { AccountModel } from '@server/models/account/account' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MAbuseFull, MAbuseMessage, MAccountDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -type NewAbuseMessagePayload = { - abuse: MAbuseFull - message: MAbuseMessage -} - -export abstract class AbstractNewAbuseMessage extends AbstractNotification { - protected messageAccount: MAccountDefault - - async loadMessageAccount () { - this.messageAccount = await AccountModel.load(this.message.accountId) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.abuseNewMessage - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.ABUSE_NEW_MESSAGE, - userId: user.id, - abuseId: this.abuse.id - }) - notification.Abuse = this.abuse - - return notification - } - - protected createEmailFor (to: string, target: 'moderator' | 'reporter') { - const text = 'New message on report #' + this.abuse.id - const abuseUrl = target === 'moderator' - ? WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.abuse.id - : WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id - - const action = { - text: 'View report #' + this.abuse.id, - url: abuseUrl - } - - return { - template: 'abuse-new-message', - to, - subject: text, - locals: { - abuseId: this.abuse.id, - abuseUrl: action.url, - messageAccountName: this.messageAccount.getDisplayName(), - messageText: this.message.message, - action - } - } - } - - protected get abuse () { - return this.payload.abuse - } - - protected get message () { - return this.payload.message - } -} diff --git a/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts b/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts deleted file mode 100644 index 97e896c6a..000000000 --- a/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { AbuseState, UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class AbuseStateChangeForReporter extends AbstractNotification { - - private user: MUserDefault - - async prepare () { - const reporter = this.abuse.ReporterAccount - if (reporter.isOwned() !== true) return - - this.user = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) - } - - log () { - logger.info('Notifying reporter of abuse % of state change.', getAbuseTargetUrl(this.abuse)) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.abuseStateChange - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.ABUSE_STATE_CHANGE, - userId: user.id, - abuseId: this.abuse.id - }) - notification.Abuse = this.abuse - - return notification - } - - createEmail (to: string) { - const text = this.abuse.state === AbuseState.ACCEPTED - ? 'Report #' + this.abuse.id + ' has been accepted' - : 'Report #' + this.abuse.id + ' has been rejected' - - const abuseUrl = WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id - - const action = { - text: 'View report #' + this.abuse.id, - url: abuseUrl - } - - return { - template: 'abuse-state-change', - to, - subject: text, - locals: { - action, - abuseId: this.abuse.id, - abuseUrl, - isAccepted: this.abuse.state === AbuseState.ACCEPTED - } - } - } - - private get abuse () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/abuse/index.ts b/server/lib/notifier/shared/abuse/index.ts deleted file mode 100644 index 7b54c5591..000000000 --- a/server/lib/notifier/shared/abuse/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './abuse-state-change-for-reporter' -export * from './new-abuse-for-moderators' -export * from './new-abuse-message-for-reporter' -export * from './new-abuse-message-for-moderators' diff --git a/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts b/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts deleted file mode 100644 index 7d86fb55f..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserAbuse, UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export type NewAbusePayload = { abuse: UserAbuse, abuseInstance: MAbuseFull, reporter: string } - -export class NewAbuseForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) - } - - log () { - logger.info('Notifying %s user/moderators of new abuse %s.', this.moderators.length, getAbuseTargetUrl(this.payload.abuseInstance)) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.abuseAsModerator - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_ABUSE_FOR_MODERATORS, - userId: user.id, - abuseId: this.payload.abuseInstance.id - }) - notification.Abuse = this.payload.abuseInstance - - return notification - } - - createEmail (to: string) { - const abuseInstance = this.payload.abuseInstance - - if (abuseInstance.VideoAbuse) return this.createVideoAbuseEmail(to) - if (abuseInstance.VideoCommentAbuse) return this.createCommentAbuseEmail(to) - - return this.createAccountAbuseEmail(to) - } - - private createVideoAbuseEmail (to: string) { - const video = this.payload.abuseInstance.VideoAbuse.Video - const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() - - return { - template: 'video-abuse-new', - to, - subject: `New video abuse report from ${this.payload.reporter}`, - locals: { - videoUrl, - isLocal: video.remote === false, - videoCreatedAt: new Date(video.createdAt).toLocaleString(), - videoPublishedAt: new Date(video.publishedAt).toLocaleString(), - videoName: video.name, - reason: this.payload.abuse.reason, - videoChannel: this.payload.abuse.video.channel, - reporter: this.payload.reporter, - action: this.buildEmailAction() - } - } - } - - private createCommentAbuseEmail (to: string) { - const comment = this.payload.abuseInstance.VideoCommentAbuse.VideoComment - const commentUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() + ';threadId=' + comment.getThreadId() - - return { - template: 'video-comment-abuse-new', - to, - subject: `New comment abuse report from ${this.payload.reporter}`, - locals: { - commentUrl, - videoName: comment.Video.name, - isLocal: comment.isOwned(), - commentCreatedAt: new Date(comment.createdAt).toLocaleString(), - reason: this.payload.abuse.reason, - flaggedAccount: this.payload.abuseInstance.FlaggedAccount.getDisplayName(), - reporter: this.payload.reporter, - action: this.buildEmailAction() - } - } - } - - private createAccountAbuseEmail (to: string) { - const account = this.payload.abuseInstance.FlaggedAccount - const accountUrl = account.getClientUrl() - - return { - template: 'account-abuse-new', - to, - subject: `New account abuse report from ${this.payload.reporter}`, - locals: { - accountUrl, - accountDisplayName: account.getDisplayName(), - isLocal: account.isOwned(), - reason: this.payload.abuse.reason, - reporter: this.payload.reporter, - action: this.buildEmailAction() - } - } - } - - private buildEmailAction () { - return { - text: 'View report #' + this.payload.abuseInstance.id, - url: WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.payload.abuseInstance.id - } - } -} diff --git a/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts b/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts deleted file mode 100644 index 9d0629690..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { MUserDefault } from '@server/types/models' -import { UserRight } from '@shared/models' -import { AbstractNewAbuseMessage } from './abstract-new-abuse-message' - -export class NewAbuseMessageForModerators extends AbstractNewAbuseMessage { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) - - // Don't notify my own message - this.moderators = this.moderators.filter(m => m.Account.id !== this.message.accountId) - if (this.moderators.length === 0) return - - await this.loadMessageAccount() - } - - log () { - logger.info('Notifying moderators of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) - } - - getTargetUsers () { - return this.moderators - } - - createEmail (to: string) { - return this.createEmailFor(to, 'moderator') - } -} diff --git a/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts b/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts deleted file mode 100644 index c5bbb5447..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { MUserDefault } from '@server/types/models' -import { AbstractNewAbuseMessage } from './abstract-new-abuse-message' - -export class NewAbuseMessageForReporter extends AbstractNewAbuseMessage { - private reporter: MUserDefault - - async prepare () { - // Only notify our users - if (this.abuse.ReporterAccount.isOwned() !== true) return - - await this.loadMessageAccount() - - const reporter = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) - // Don't notify my own message - if (reporter.Account.id === this.message.accountId) return - - this.reporter = reporter - } - - log () { - logger.info('Notifying reporter of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) - } - - getTargetUsers () { - if (!this.reporter) return [] - - return [ this.reporter ] - } - - createEmail (to: string) { - return this.createEmailFor(to, 'reporter') - } -} diff --git a/server/lib/notifier/shared/blacklist/index.ts b/server/lib/notifier/shared/blacklist/index.ts deleted file mode 100644 index 2f98d88ae..000000000 --- a/server/lib/notifier/shared/blacklist/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './new-auto-blacklist-for-moderators' -export * from './new-blacklist-for-owner' -export * from './unblacklist-for-owner' diff --git a/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts b/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts deleted file mode 100644 index ad2cc00ea..000000000 --- a/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MUserDefault, MUserWithNotificationSetting, MVideoBlacklistLightVideo, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewAutoBlacklistForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_BLACKLIST) - } - - log () { - logger.info('Notifying %s moderators of video auto-blacklist %s.', this.moderators.length, this.payload.Video.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.videoAutoBlacklistAsModerator - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS, - userId: user.id, - videoBlacklistId: this.payload.id - }) - notification.VideoBlacklist = this.payload - - return notification - } - - async createEmail (to: string) { - const videoAutoBlacklistUrl = WEBSERVER.URL + '/admin/moderation/video-auto-blacklist/list' - const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() - const channel = await VideoChannelModel.loadAndPopulateAccount(this.payload.Video.channelId) - - return { - template: 'video-auto-blacklist-new', - to, - subject: 'A new video is pending moderation', - locals: { - channel: channel.toFormattedSummaryJSON(), - videoUrl, - videoName: this.payload.Video.name, - action: { - text: 'Review autoblacklist', - url: videoAutoBlacklistUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts b/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts deleted file mode 100644 index 342b69ec7..000000000 --- a/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoBlacklistVideo, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewBlacklistForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.videoId) - } - - log () { - logger.info('Notifying user %s that its video %s has been blacklisted.', this.user.username, this.payload.Video.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.blacklistOnMyVideo - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.BLACKLIST_ON_MY_VIDEO, - userId: user.id, - videoBlacklistId: this.payload.id - }) - notification.VideoBlacklist = this.payload - - return notification - } - - createEmail (to: string) { - const videoName = this.payload.Video.name - const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() - - const reasonString = this.payload.reason ? ` for the following reason: ${this.payload.reason}` : '' - const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.INSTANCE.NAME} has been blacklisted${reasonString}.` - - return { - to, - subject: `Video ${videoName} blacklisted`, - text: blockedString, - locals: { - title: 'Your video was blacklisted' - } - } - } -} diff --git a/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts b/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts deleted file mode 100644 index e6f90e23c..000000000 --- a/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class UnblacklistForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.id) - } - - log () { - logger.info('Notifying user %s that its video %s has been unblacklisted.', this.user.username, this.payload.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.blacklistOnMyVideo - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.UNBLACKLIST_ON_MY_VIDEO, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const video = this.payload - const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() - - return { - to, - subject: `Video ${video.name} unblacklisted`, - text: `Your video "${video.name}" (${videoUrl}) on ${CONFIG.INSTANCE.NAME} has been unblacklisted.`, - locals: { - title: 'Your video was unblacklisted' - } - } - } -} diff --git a/server/lib/notifier/shared/comment/comment-mention.ts b/server/lib/notifier/shared/comment/comment-mention.ts deleted file mode 100644 index 3074e97db..000000000 --- a/server/lib/notifier/shared/comment/comment-mention.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { toSafeHtml } from '@server/helpers/markdown' -import { WEBSERVER } from '@server/initializers/constants' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { getServerActor } from '@server/models/application/application' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { - MCommentOwnerVideo, - MUserDefault, - MUserNotifSettingAccount, - MUserWithNotificationSetting, - UserNotificationModelForApi -} from '@server/types/models' -import { UserNotificationSettingValue, UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common' - -export class CommentMention extends AbstractNotification { - private users: MUserDefault[] - - private serverAccountId: number - - private accountMutedHash: { [ id: number ]: boolean } - private instanceMutedHash: { [ id: number ]: boolean } - - async prepare () { - const extractedUsernames = this.payload.extractMentions() - logger.debug( - 'Extracted %d username from comment %s.', extractedUsernames.length, this.payload.url, - { usernames: extractedUsernames, text: this.payload.text } - ) - - this.users = await UserModel.listByUsernames(extractedUsernames) - - if (this.payload.Video.isOwned()) { - const userException = await UserModel.loadByVideoId(this.payload.videoId) - this.users = this.users.filter(u => u.id !== userException.id) - } - - // Don't notify if I mentioned myself - this.users = this.users.filter(u => u.Account.id !== this.payload.accountId) - - if (this.users.length === 0) return - - this.serverAccountId = (await getServerActor()).Account.id - - const sourceAccounts = this.users.map(u => u.Account.id).concat([ this.serverAccountId ]) - - this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, this.payload.accountId) - this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, this.payload.Account.Actor.serverId) - } - - log () { - logger.info('Notifying %d users of new comment %s.', this.users.length, this.payload.url) - } - - getSetting (user: MUserNotifSettingAccount) { - const accountId = user.Account.id - if ( - this.accountMutedHash[accountId] === true || this.instanceMutedHash[accountId] === true || - this.accountMutedHash[this.serverAccountId] === true || this.instanceMutedHash[this.serverAccountId] === true - ) { - return UserNotificationSettingValue.NONE - } - - return user.NotificationSetting.commentMention - } - - getTargetUsers () { - return this.users - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.COMMENT_MENTION, - userId: user.id, - commentId: this.payload.id - }) - notification.VideoComment = this.payload - - return notification - } - - createEmail (to: string) { - const comment = this.payload - - const accountName = comment.Account.getDisplayName() - const video = comment.Video - const videoUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() - const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath() - const commentHtml = toSafeHtml(comment.text) - - return { - template: 'video-comment-mention', - to, - subject: 'Mention on video ' + video.name, - locals: { - comment, - commentHtml, - video, - videoUrl, - accountName, - action: { - text: 'View comment', - url: commentUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/comment/index.ts b/server/lib/notifier/shared/comment/index.ts deleted file mode 100644 index ae01a9646..000000000 --- a/server/lib/notifier/shared/comment/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './comment-mention' -export * from './new-comment-for-video-owner' diff --git a/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts b/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts deleted file mode 100644 index 4f96439a3..000000000 --- a/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { toSafeHtml } from '@server/helpers/markdown' -import { WEBSERVER } from '@server/initializers/constants' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MCommentOwnerVideo, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewCommentForVideoOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.videoId) - } - - log () { - logger.info('Notifying owner of a video %s of new comment %s.', this.user.username, this.payload.url) - } - - isDisabled () { - if (this.payload.Video.isOwned() === false) return true - - // Not our user or user comments its own video - if (!this.user || this.payload.Account.userId === this.user.id) return true - - return isBlockedByServerOrAccount(this.payload.Account, this.user.Account) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newCommentOnMyVideo - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, - userId: user.id, - commentId: this.payload.id - }) - notification.VideoComment = this.payload - - return notification - } - - createEmail (to: string) { - const video = this.payload.Video - const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() - const commentUrl = WEBSERVER.URL + this.payload.getCommentStaticPath() - const commentHtml = toSafeHtml(this.payload.text) - - return { - template: 'video-comment-new', - to, - subject: 'New comment on your video ' + video.name, - locals: { - accountName: this.payload.Account.getDisplayName(), - accountUrl: this.payload.Account.Actor.url, - comment: this.payload, - commentHtml, - video, - videoUrl, - action: { - text: 'View comment', - url: commentUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/common/abstract-notification.ts b/server/lib/notifier/shared/common/abstract-notification.ts deleted file mode 100644 index 79403611e..000000000 --- a/server/lib/notifier/shared/common/abstract-notification.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { EmailPayload, UserNotificationSettingValue } from '@shared/models' - -export abstract class AbstractNotification { - - constructor (protected readonly payload: T) { - - } - - abstract prepare (): Promise - abstract log (): void - - abstract getSetting (user: U): UserNotificationSettingValue - abstract getTargetUsers (): U[] - - abstract createNotification (user: U): UserNotificationModelForApi - abstract createEmail (to: string): EmailPayload | Promise - - isDisabled (): boolean | Promise { - return false - } - -} diff --git a/server/lib/notifier/shared/common/index.ts b/server/lib/notifier/shared/common/index.ts deleted file mode 100644 index 0b2570278..000000000 --- a/server/lib/notifier/shared/common/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './abstract-notification' diff --git a/server/lib/notifier/shared/follow/auto-follow-for-instance.ts b/server/lib/notifier/shared/follow/auto-follow-for-instance.ts deleted file mode 100644 index ab9747ba8..000000000 --- a/server/lib/notifier/shared/follow/auto-follow-for-instance.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class AutoFollowForInstance extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) - } - - log () { - logger.info('Notifying %d administrators of auto instance following: %s.', this.admins.length, this.actorFollow.ActorFollowing.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.autoInstanceFollowing - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.AUTO_INSTANCE_FOLLOWING, - userId: user.id, - actorFollowId: this.actorFollow.id - }) - notification.ActorFollow = this.actorFollow - - return notification - } - - createEmail (to: string) { - const instanceUrl = this.actorFollow.ActorFollowing.url - - return { - to, - subject: 'Auto instance following', - text: `Your instance automatically followed a new instance: ${instanceUrl}.` - } - } - - private get actorFollow () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/follow/follow-for-instance.ts b/server/lib/notifier/shared/follow/follow-for-instance.ts deleted file mode 100644 index 777a12ef4..000000000 --- a/server/lib/notifier/shared/follow/follow-for-instance.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class FollowForInstance extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) - } - - isDisabled () { - const follower = Object.assign(this.actorFollow.ActorFollower.Account, { Actor: this.actorFollow.ActorFollower }) - - return isBlockedByServerOrAccount(follower) - } - - log () { - logger.info('Notifying %d administrators of new instance follower: %s.', this.admins.length, this.actorFollow.ActorFollower.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newInstanceFollower - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_INSTANCE_FOLLOWER, - userId: user.id, - actorFollowId: this.actorFollow.id - }) - notification.ActorFollow = this.actorFollow - - return notification - } - - createEmail (to: string) { - const awaitingApproval = this.actorFollow.state === 'pending' - ? ' awaiting manual approval.' - : '' - - return { - to, - subject: 'New instance follower', - text: `Your instance has a new follower: ${this.actorFollow.ActorFollower.url}${awaitingApproval}.`, - locals: { - title: 'New instance follower', - action: { - text: 'Review followers', - url: WEBSERVER.URL + '/admin/follows/followers-list' - } - } - } - } - - private get actorFollow () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/follow/follow-for-user.ts b/server/lib/notifier/shared/follow/follow-for-user.ts deleted file mode 100644 index 697c82cdd..000000000 --- a/server/lib/notifier/shared/follow/follow-for-user.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class FollowForUser extends AbstractNotification { - private followType: 'account' | 'channel' - private user: MUserDefault - - async prepare () { - // Account follows one of our account? - this.followType = 'channel' - this.user = await UserModel.loadByChannelActorId(this.actorFollow.ActorFollowing.id) - - // Account follows one of our channel? - if (!this.user) { - this.user = await UserModel.loadByAccountActorId(this.actorFollow.ActorFollowing.id) - this.followType = 'account' - } - } - - async isDisabled () { - if (this.payload.ActorFollowing.isOwned() === false) return true - - const followerAccount = this.actorFollow.ActorFollower.Account - const followerAccountWithActor = Object.assign(followerAccount, { Actor: this.actorFollow.ActorFollower }) - - return isBlockedByServerOrAccount(followerAccountWithActor, this.user.Account) - } - - log () { - logger.info('Notifying user %s of new follower: %s.', this.user.username, this.actorFollow.ActorFollower.Account.getDisplayName()) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newFollow - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_FOLLOW, - userId: user.id, - actorFollowId: this.actorFollow.id - }) - notification.ActorFollow = this.actorFollow - - return notification - } - - createEmail (to: string) { - const following = this.actorFollow.ActorFollowing - const follower = this.actorFollow.ActorFollower - - const followingName = (following.VideoChannel || following.Account).getDisplayName() - - return { - template: 'follower-on-channel', - to, - subject: `New follower on your channel ${followingName}`, - locals: { - followerName: follower.Account.getDisplayName(), - followerUrl: follower.url, - followingName, - followingUrl: following.url, - followType: this.followType - } - } - } - - private get actorFollow () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/follow/index.ts b/server/lib/notifier/shared/follow/index.ts deleted file mode 100644 index 27f5289d9..000000000 --- a/server/lib/notifier/shared/follow/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './auto-follow-for-instance' -export * from './follow-for-instance' -export * from './follow-for-user' diff --git a/server/lib/notifier/shared/index.ts b/server/lib/notifier/shared/index.ts deleted file mode 100644 index cc3ce8c7c..000000000 --- a/server/lib/notifier/shared/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './abuse' -export * from './blacklist' -export * from './comment' -export * from './common' -export * from './follow' -export * from './instance' -export * from './video-publication' diff --git a/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts b/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts deleted file mode 100644 index 5044f2068..000000000 --- a/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class DirectRegistrationForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_USERS) - } - - log () { - logger.info('Notifying %s moderators of new user registration of %s.', this.moderators.length, this.payload.username) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newUserRegistration - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_USER_REGISTRATION, - userId: user.id, - accountId: this.payload.Account.id - }) - notification.Account = this.payload.Account - - return notification - } - - createEmail (to: string) { - return { - template: 'user-registered', - to, - subject: `A new user registered on ${CONFIG.INSTANCE.NAME}: ${this.payload.username}`, - locals: { - user: this.payload - } - } - } -} diff --git a/server/lib/notifier/shared/instance/index.ts b/server/lib/notifier/shared/instance/index.ts deleted file mode 100644 index 8c75a8ee9..000000000 --- a/server/lib/notifier/shared/instance/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './new-peertube-version-for-admins' -export * from './new-plugin-version-for-admins' -export * from './direct-registration-for-moderators' -export * from './registration-request-for-moderators' diff --git a/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts b/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts deleted file mode 100644 index f5646c666..000000000 --- a/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MApplication, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export type NewPeerTubeVersionForAdminsPayload = { - application: MApplication - latestVersion: string -} - -export class NewPeerTubeVersionForAdmins extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - // Use the debug right to know who is an administrator - this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) - } - - log () { - logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newPeerTubeVersion - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_PEERTUBE_VERSION, - userId: user.id, - applicationId: this.payload.application.id - }) - notification.Application = this.payload.application - - return notification - } - - createEmail (to: string) { - return { - to, - template: 'peertube-version-new', - subject: `A new PeerTube version is available: ${this.payload.latestVersion}`, - locals: { - latestVersion: this.payload.latestVersion - } - } - } -} diff --git a/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts b/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts deleted file mode 100644 index 547c6726c..000000000 --- a/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MPlugin, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewPluginVersionForAdmins extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - // Use the debug right to know who is an administrator - this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) - } - - log () { - logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newPluginVersion - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_PLUGIN_VERSION, - userId: user.id, - pluginId: this.plugin.id - }) - notification.Plugin = this.plugin - - return notification - } - - createEmail (to: string) { - const pluginUrl = WEBSERVER.URL + '/admin/plugins/list-installed?pluginType=' + this.plugin.type - - return { - to, - template: 'plugin-version-new', - subject: `A new plugin/theme version is available: ${this.plugin.name}@${this.plugin.latestVersion}`, - locals: { - pluginName: this.plugin.name, - latestVersion: this.plugin.latestVersion, - pluginUrl - } - } - } - - private get plugin () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/instance/registration-request-for-moderators.ts b/server/lib/notifier/shared/instance/registration-request-for-moderators.ts deleted file mode 100644 index 79920245a..000000000 --- a/server/lib/notifier/shared/instance/registration-request-for-moderators.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MRegistration, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class RegistrationRequestForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_REGISTRATIONS) - } - - log () { - logger.info('Notifying %s moderators of new user registration request of %s.', this.moderators.length, this.payload.username) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newUserRegistration - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_USER_REGISTRATION_REQUEST, - userId: user.id, - userRegistrationId: this.payload.id - }) - notification.UserRegistration = this.payload - - return notification - } - - createEmail (to: string) { - return { - template: 'user-registration-request', - to, - subject: `A new user wants to register: ${this.payload.username}`, - locals: { - registration: this.payload - } - } - } -} diff --git a/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts b/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts deleted file mode 100644 index a940cde69..000000000 --- a/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export abstract class AbstractOwnedVideoPublication extends AbstractNotification { - protected user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.id) - } - - log () { - logger.info('Notifying user %s of the publication of its video %s.', this.user.username, this.payload.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.myVideoPublished - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.MY_VIDEO_PUBLISHED, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() - - return { - to, - subject: `Your video ${this.payload.name} has been published`, - text: `Your video "${this.payload.name}" has been published.`, - locals: { - title: 'Your video is live', - action: { - text: 'View video', - url: videoUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts b/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts deleted file mode 100644 index 3bd64692f..000000000 --- a/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoImportVideo, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export type ImportFinishedForOwnerPayload = { - videoImport: MVideoImportVideo - success: boolean -} - -export class ImportFinishedForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoImportId(this.videoImport.id) - } - - log () { - logger.info('Notifying user %s its video import %s is finished.', this.user.username, this.videoImport.getTargetIdentifier()) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.myVideoImportFinished - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: this.payload.success - ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS - : UserNotificationType.MY_VIDEO_IMPORT_ERROR, - - userId: user.id, - videoImportId: this.videoImport.id - }) - notification.VideoImport = this.videoImport - - return notification - } - - createEmail (to: string) { - if (this.payload.success) return this.createSuccessEmail(to) - - return this.createFailEmail(to) - } - - private createSuccessEmail (to: string) { - const videoUrl = WEBSERVER.URL + this.videoImport.Video.getWatchStaticPath() - - return { - to, - subject: `Your video import ${this.videoImport.getTargetIdentifier()} is complete`, - text: `Your video "${this.videoImport.getTargetIdentifier()}" just finished importing.`, - locals: { - title: 'Import complete', - action: { - text: 'View video', - url: videoUrl - } - } - } - } - - private createFailEmail (to: string) { - const importUrl = WEBSERVER.URL + '/my-library/video-imports' - - const text = - `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error.` + - '\n\n' + - `See your videos import dashboard for more information: ${importUrl}.` - - return { - to, - subject: `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error`, - text, - locals: { - title: 'Import failed', - action: { - text: 'Review imports', - url: importUrl - } - } - } - } - - private get videoImport () { - return this.payload.videoImport - } -} diff --git a/server/lib/notifier/shared/video-publication/index.ts b/server/lib/notifier/shared/video-publication/index.ts deleted file mode 100644 index 5e92cb011..000000000 --- a/server/lib/notifier/shared/video-publication/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './new-video-for-subscribers' -export * from './import-finished-for-owner' -export * from './owned-publication-after-auto-unblacklist' -export * from './owned-publication-after-schedule-update' -export * from './owned-publication-after-transcoding' -export * from './studio-edition-finished-for-owner' diff --git a/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts b/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts deleted file mode 100644 index df7a5561d..000000000 --- a/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserWithNotificationSetting, MVideoAccountLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, VideoPrivacy, VideoState } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewVideoForSubscribers extends AbstractNotification { - private users: MUserWithNotificationSetting[] - - async prepare () { - // List all followers that are users - this.users = await UserModel.listUserSubscribersOf(this.payload.VideoChannel.actorId) - } - - log () { - logger.info('Notifying %d users of new video %s.', this.users.length, this.payload.url) - } - - isDisabled () { - return this.payload.privacy !== VideoPrivacy.PUBLIC || this.payload.state !== VideoState.PUBLISHED || this.payload.isBlacklisted() - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newVideoFromSubscription - } - - getTargetUsers () { - return this.users - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const channelName = this.payload.VideoChannel.getDisplayName() - const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() - - return { - to, - subject: channelName + ' just published a new video', - text: `Your subscription ${channelName} just published a new video: "${this.payload.name}".`, - locals: { - title: 'New content ', - action: { - text: 'View video', - url: videoUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts deleted file mode 100644 index 27d89a5c7..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts +++ /dev/null @@ -1,11 +0,0 @@ - -import { VideoState } from '@shared/models' -import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' - -export class OwnedPublicationAfterAutoUnblacklist extends AbstractOwnedVideoPublication { - - isDisabled () { - // Don't notify if video is still waiting for transcoding or scheduled update - return !!this.payload.ScheduleVideoUpdate || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) - } -} diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts deleted file mode 100644 index 2e253b358..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { VideoState } from '@shared/models' -import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' - -export class OwnedPublicationAfterScheduleUpdate extends AbstractOwnedVideoPublication { - - isDisabled () { - // Don't notify if video is still blacklisted or waiting for transcoding - return !!this.payload.VideoBlacklist || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) - } -} diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts deleted file mode 100644 index 4fab1090f..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' - -export class OwnedPublicationAfterTranscoding extends AbstractOwnedVideoPublication { - - isDisabled () { - // Don't notify if didn't wait for transcoding or video is still blacklisted/waiting for scheduled update - return !this.payload.waitTranscoding || !!this.payload.VideoBlacklist || !!this.payload.ScheduleVideoUpdate - } -} diff --git a/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts b/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts deleted file mode 100644 index f36399f05..000000000 --- a/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class StudioEditionFinishedForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.id) - } - - log () { - logger.info('Notifying user %s its video studio edition %s is finished.', this.user.username, this.payload.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.myVideoStudioEditionFinished - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.MY_VIDEO_STUDIO_EDITION_FINISHED, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() - - return { - to, - subject: `Edition of your video ${this.payload.name} has finished`, - text: `Edition of your video ${this.payload.name} has finished.`, - locals: { - title: 'Video edition has finished', - action: { - text: 'View video', - url: videoUrl - } - } - } - } -} diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts deleted file mode 100644 index 3ad6cab63..000000000 --- a/server/lib/object-storage/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './keys' -export * from './proxy' -export * from './pre-signed-urls' -export * from './urls' -export * from './videos' diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts deleted file mode 100644 index 6d2098298..000000000 --- a/server/lib/object-storage/keys.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { join } from 'path' -import { MStreamingPlaylistVideo } from '@server/types/models' - -function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { - return join(generateHLSObjectBaseStorageKey(playlist), filename) -} - -function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { - return join(playlist.getStringType(), playlist.Video.uuid) -} - -function generateWebVideoObjectStorageKey (filename: string) { - return filename -} - -export { - generateHLSObjectStorageKey, - generateHLSObjectBaseStorageKey, - generateWebVideoObjectStorageKey -} diff --git a/server/lib/object-storage/pre-signed-urls.ts b/server/lib/object-storage/pre-signed-urls.ts deleted file mode 100644 index caf149bb8..000000000 --- a/server/lib/object-storage/pre-signed-urls.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { GetObjectCommand } from '@aws-sdk/client-s3' -import { getSignedUrl } from '@aws-sdk/s3-request-presigner' -import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' -import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' -import { buildKey, getClient } from './shared' -import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls' - -export async function generateWebVideoPresignedUrl (options: { - file: MVideoFile - downloadFilename: string -}) { - const { file, downloadFilename } = options - - const key = generateWebVideoObjectStorageKey(file.filename) - - const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS), - ResponseContentDisposition: `attachment; filename=${downloadFilename}` - }) - - const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) - - return getWebVideoPublicFileUrl(url) -} - -export async function generateHLSFilePresignedUrl (options: { - streamingPlaylist: MStreamingPlaylistVideo - file: MVideoFile - downloadFilename: string -}) { - const { streamingPlaylist, file, downloadFilename } = options - - const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename) - - const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), - ResponseContentDisposition: `attachment; filename=${downloadFilename}` - }) - - const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) - - return getHLSPublicFileUrl(url) -} diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts deleted file mode 100644 index c09a0d1b0..000000000 --- a/server/lib/object-storage/proxy.ts +++ /dev/null @@ -1,97 +0,0 @@ -import express from 'express' -import { PassThrough, pipeline } from 'stream' -import { GetObjectCommandOutput } from '@aws-sdk/client-s3' -import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist' -import { logger } from '@server/helpers/logger' -import { StreamReplacer } from '@server/helpers/stream-replacer' -import { MStreamingPlaylist, MVideo } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { injectQueryToPlaylistUrls } from '../hls' -import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos' - -export async function proxifyWebVideoFile (options: { - req: express.Request - res: express.Response - filename: string -}) { - const { req, res, filename } = options - - logger.debug('Proxifying Web Video file %s from object storage.', filename) - - try { - const { response: s3Response, stream } = await getWebVideoFileReadStream({ - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - return stream.pipe(res) - } catch (err) { - return handleObjectStorageFailure(res, err) - } -} - -export async function proxifyHLS (options: { - req: express.Request - res: express.Response - playlist: MStreamingPlaylist - video: MVideo - filename: string - reinjectVideoFileToken: boolean -}) { - const { req, res, playlist, video, filename, reinjectVideoFileToken } = options - - logger.debug('Proxifying HLS file %s from object storage.', filename) - - try { - const { response: s3Response, stream } = await getHLSFileReadStream({ - playlist: playlist.withVideo(video), - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - const streamReplacer = reinjectVideoFileToken - ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) - : new PassThrough() - - return pipeline( - stream, - streamReplacer, - res, - err => { - if (!err) return - - handleObjectStorageFailure(res, err) - } - ) - } catch (err) { - return handleObjectStorageFailure(res, err) - } -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function handleObjectStorageFailure (res: express.Response, err: Error) { - if (err.name === 'NoSuchKey') { - logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) - return res.sendStatus(HttpStatusCode.NOT_FOUND_404) - } - - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: err.message, - type: err.name - }) -} - -function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { - if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { - res.setHeader('Content-Range', s3Response.ContentRange) - res.status(HttpStatusCode.PARTIAL_CONTENT_206) - } -} diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts deleted file mode 100644 index d5cb074df..000000000 --- a/server/lib/object-storage/shared/client.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { S3Client } from '@aws-sdk/client-s3' -import { NodeHttpHandler } from '@aws-sdk/node-http-handler' -import { logger } from '@server/helpers/logger' -import { isProxyEnabled } from '@server/helpers/proxy' -import { getAgent } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { lTags } from './logger' - -function getProxyRequestHandler () { - if (!isProxyEnabled()) return null - - const { agent } = getAgent() - - return new NodeHttpHandler({ - httpAgent: agent.http, - httpsAgent: agent.https - }) -} - -let endpointParsed: URL -function getEndpointParsed () { - if (endpointParsed) return endpointParsed - - endpointParsed = new URL(getEndpoint()) - - return endpointParsed -} - -let s3Client: S3Client -function getClient () { - if (s3Client) return s3Client - - const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE - - s3Client = new S3Client({ - endpoint: getEndpoint(), - region: OBJECT_STORAGE.REGION, - credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID - ? { - accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, - secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY - } - : undefined, - requestHandler: getProxyRequestHandler() - }) - - logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) - - return s3Client -} - -// --------------------------------------------------------------------------- - -export { - getEndpointParsed, - getClient -} - -// --------------------------------------------------------------------------- - -let endpoint: string -function getEndpoint () { - if (endpoint) return endpoint - - const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT - endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') - ? CONFIG.OBJECT_STORAGE.ENDPOINT - : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT - - return endpoint -} diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts deleted file mode 100644 index 11e10aa9f..000000000 --- a/server/lib/object-storage/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './client' -export * from './logger' -export * from './object-storage-helpers' diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts deleted file mode 100644 index 8ab7cbd71..000000000 --- a/server/lib/object-storage/shared/logger.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loggerTagsFactory } from '@server/helpers/logger' - -const lTags = loggerTagsFactory('object-storage') - -export { - lTags -} diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts deleted file mode 100644 index 0d8878bd2..000000000 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { map } from 'bluebird' -import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' -import { dirname } from 'path' -import { Readable } from 'stream' -import { - _Object, - CompleteMultipartUploadCommandOutput, - DeleteObjectCommand, - GetObjectCommand, - ListObjectsV2Command, - PutObjectAclCommand, - PutObjectCommandInput, - S3Client -} from '@aws-sdk/client-s3' -import { Upload } from '@aws-sdk/lib-storage' -import { pipelinePromise } from '@server/helpers/core-utils' -import { isArray } from '@server/helpers/custom-validators/misc' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { getInternalUrl } from '../urls' -import { getClient } from './client' -import { lTags } from './logger' - -type BucketInfo = { - BUCKET_NAME: string - PREFIX?: string -} - -async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { - const s3Client = getClient() - - const commandPrefix = bucketInfo.PREFIX + prefix - const listCommand = new ListObjectsV2Command({ - Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix - }) - - const listedObjects = await s3Client.send(listCommand) - - if (isArray(listedObjects.Contents) !== true) return [] - - return listedObjects.Contents.map(c => c.Key) -} - -// --------------------------------------------------------------------------- - -async function storeObject (options: { - inputPath: string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}): Promise { - const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options - - logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - - const fileStream = createReadStream(inputPath) - - return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) -} - -async function storeContent (options: { - content: string - inputPath: string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}): Promise { - const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options - - logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - - return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) -} - -// --------------------------------------------------------------------------- - -async function updateObjectACL (options: { - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { objectStorageKey, bucketInfo, isPrivate } = options - - const acl = getACL(isPrivate) - if (!acl) return - - const key = buildKey(objectStorageKey, bucketInfo) - - logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) - - const command = new PutObjectAclCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: key, - ACL: acl - }) - - await getClient().send(command) -} - -function updatePrefixACL (options: { - prefix: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { prefix, bucketInfo, isPrivate } = options - - const acl = getACL(isPrivate) - if (!acl) return - - logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) - - return applyOnPrefix({ - prefix, - bucketInfo, - commandBuilder: obj => { - logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) - - return new PutObjectAclCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: obj.Key, - ACL: acl - }) - } - }) -} - -// --------------------------------------------------------------------------- - -function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { - const key = buildKey(objectStorageKey, bucketInfo) - - return removeObjectByFullKey(key, bucketInfo) -} - -function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { - logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) - - const command = new DeleteObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: fullKey - }) - - return getClient().send(command) -} - -async function removePrefix (prefix: string, bucketInfo: BucketInfo) { - logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) - - return applyOnPrefix({ - prefix, - bucketInfo, - commandBuilder: obj => { - logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) - - return new DeleteObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: obj.Key - }) - } - }) -} - -// --------------------------------------------------------------------------- - -async function makeAvailable (options: { - key: string - destination: string - bucketInfo: BucketInfo -}) { - const { key, destination, bucketInfo } = options - - await ensureDir(dirname(options.destination)) - - const command = new GetObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(key, bucketInfo) - }) - const response = await getClient().send(command) - - const file = createWriteStream(destination) - await pipelinePromise(response.Body as Readable, file) - - file.close() -} - -function buildKey (key: string, bucketInfo: BucketInfo) { - return bucketInfo.PREFIX + key -} - -// --------------------------------------------------------------------------- - -async function createObjectReadStream (options: { - key: string - bucketInfo: BucketInfo - rangeHeader: string -}) { - const { key, bucketInfo, rangeHeader } = options - - const command = new GetObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(key, bucketInfo), - Range: rangeHeader - }) - - const response = await getClient().send(command) - - return { - response, - stream: response.Body as Readable - } -} - -// --------------------------------------------------------------------------- - -export { - BucketInfo, - buildKey, - - storeObject, - storeContent, - - removeObject, - removeObjectByFullKey, - removePrefix, - - makeAvailable, - - updateObjectACL, - updatePrefixACL, - - listKeysOfPrefix, - createObjectReadStream -} - -// --------------------------------------------------------------------------- - -async function uploadToStorage (options: { - content: ReadStream | string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { content, objectStorageKey, bucketInfo, isPrivate } = options - - const input: PutObjectCommandInput = { - Body: content, - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(objectStorageKey, bucketInfo) - } - - const acl = getACL(isPrivate) - if (acl) input.ACL = acl - - const parallelUploads3 = new Upload({ - client: getClient(), - queueSize: 4, - partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART, - - // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts - // More detailed explanation: - // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274 - // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928 - leavePartsOnError: true, - params: input - }) - - const response = (await parallelUploads3.done()) as CompleteMultipartUploadCommandOutput - // Check is needed even if the HTTP status code is 200 OK - // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html - if (!response.Bucket) { - const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}` - logger.error(message, { response, ...lTags() }) - throw new Error(message) - } - - logger.debug( - 'Completed %s%s in bucket %s', - bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata } - ) - - return getInternalUrl(bucketInfo, objectStorageKey) -} - -async function applyOnPrefix (options: { - prefix: string - bucketInfo: BucketInfo - commandBuilder: (obj: _Object) => Parameters[0] - - continuationToken?: string -}) { - const { prefix, bucketInfo, commandBuilder, continuationToken } = options - - const s3Client = getClient() - - const commandPrefix = buildKey(prefix, bucketInfo) - const listCommand = new ListObjectsV2Command({ - Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix, - ContinuationToken: continuationToken - }) - - const listedObjects = await s3Client.send(listCommand) - - if (isArray(listedObjects.Contents) !== true) { - const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` - - logger.error(message, { response: listedObjects, ...lTags() }) - throw new Error(message) - } - - await map(listedObjects.Contents, object => { - const command = commandBuilder(object) - - return s3Client.send(command) - }, { concurrency: 10 }) - - // Repeat if not all objects could be listed at once (limit of 1000?) - if (listedObjects.IsTruncated) { - await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) - } -} - -function getACL (isPrivate: boolean) { - return isPrivate - ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE - : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC -} diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts deleted file mode 100644 index 40619cd5a..000000000 --- a/server/lib/object-storage/urls.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants' -import { MVideoUUID } from '@server/types/models' -import { BucketInfo, buildKey, getEndpointParsed } from './shared' - -function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { - return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) -} - -// --------------------------------------------------------------------------- - -function getWebVideoPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL - if (!baseUrl) return fileUrl - - return replaceByBaseUrl(fileUrl, baseUrl) -} - -function getHLSPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL - if (!baseUrl) return fileUrl - - return replaceByBaseUrl(fileUrl, baseUrl) -} - -// --------------------------------------------------------------------------- - -function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { - return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` -} - -function getWebVideoPrivateFileUrl (filename: string) { - return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename -} - -// --------------------------------------------------------------------------- - -export { - getInternalUrl, - - getWebVideoPublicFileUrl, - getHLSPublicFileUrl, - - getHLSPrivateFileUrl, - getWebVideoPrivateFileUrl, - - replaceByBaseUrl -} - -// --------------------------------------------------------------------------- - -function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { - if (baseUrl) return baseUrl - - return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` -} - -const regex = new RegExp('https?://[^/]+') -function replaceByBaseUrl (fileUrl: string, baseUrl: string) { - if (!fileUrl) return fileUrl - - return fileUrl.replace(regex, baseUrl) -} diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts deleted file mode 100644 index 891e9ff76..000000000 --- a/server/lib/object-storage/videos.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { basename, join } from 'path' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models' -import { getHLSDirectory } from '../paths' -import { VideoPathManager } from '../video-path-manager' -import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' -import { - createObjectReadStream, - listKeysOfPrefix, - lTags, - makeAvailable, - removeObject, - removeObjectByFullKey, - removePrefix, - storeContent, - storeObject, - updateObjectACL, - updatePrefixACL -} from './shared' - -function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { - return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -// --------------------------------------------------------------------------- - -function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { - return storeObject({ - inputPath: join(getHLSDirectory(playlist.Video), filename), - objectStorageKey: generateHLSObjectStorageKey(playlist, filename), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { - return storeObject({ - inputPath: path, - objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { - return storeContent({ - content, - inputPath: path, - objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -function storeWebVideoFile (video: MVideo, file: MVideoFile) { - return storeObject({ - inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), - objectStorageKey: generateWebVideoObjectStorageKey(file.filename), - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - isPrivate: video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { - await updateObjectACL({ - objectStorageKey: generateWebVideoObjectStorageKey(file.filename), - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - isPrivate: video.hasPrivateStaticPath() - }) -} - -async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { - await updatePrefixACL({ - prefix: generateHLSObjectBaseStorageKey(playlist), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { - return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { - return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { - return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByFullKey (key: string) { - return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -// --------------------------------------------------------------------------- - -function removeWebVideoObjectStorage (videoFile: MVideoFile) { - return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS) -} - -// --------------------------------------------------------------------------- - -async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { - const key = generateHLSObjectStorageKey(playlist, filename) - - logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) - - await makeAvailable({ - key, - destination, - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS - }) - - return destination -} - -async function makeWebVideoFileAvailable (filename: string, destination: string) { - const key = generateWebVideoObjectStorageKey(filename) - - logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags()) - - await makeAvailable({ - key, - destination, - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS - }) - - return destination -} - -// --------------------------------------------------------------------------- - -function getWebVideoFileReadStream (options: { - filename: string - rangeHeader: string -}) { - const { filename, rangeHeader } = options - - const key = generateWebVideoObjectStorageKey(filename) - - return createObjectReadStream({ - key, - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - rangeHeader - }) -} - -function getHLSFileReadStream (options: { - playlist: MStreamingPlaylistVideo - filename: string - rangeHeader: string -}) { - const { playlist, filename, rangeHeader } = options - - const key = generateHLSObjectStorageKey(playlist, filename) - - return createObjectReadStream({ - key, - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - rangeHeader - }) -} - -// --------------------------------------------------------------------------- - -export { - listHLSFileKeysOf, - - storeWebVideoFile, - storeHLSFileFromFilename, - storeHLSFileFromPath, - storeHLSFileFromContent, - - updateWebVideoFileACL, - updateHLSFilesACL, - - removeHLSObjectStorage, - removeHLSFileObjectStorageByFilename, - removeHLSFileObjectStorageByPath, - removeHLSFileObjectStorageByFullKey, - - removeWebVideoObjectStorage, - - makeWebVideoFileAvailable, - makeHLSFileAvailable, - - getWebVideoFileReadStream, - getHLSFileReadStream -} diff --git a/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts deleted file mode 100644 index ef40c0fa9..000000000 --- a/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Meter } from '@opentelemetry/api' - -export class BittorrentTrackerObserversBuilder { - - constructor (private readonly meter: Meter, private readonly trackerServer: any) { - - } - - buildObservers () { - const activeInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_active_infohashes_total', { - description: 'Total active infohashes in the PeerTube BitTorrent Tracker' - }) - const inactiveInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_inactive_infohashes_total', { - description: 'Total inactive infohashes in the PeerTube BitTorrent Tracker' - }) - const peers = this.meter.createObservableGauge('peertube_bittorrent_tracker_peers_total', { - description: 'Total peers in the PeerTube BitTorrent Tracker' - }) - - this.meter.addBatchObservableCallback(observableResult => { - const infohashes = Object.keys(this.trackerServer.torrents) - - const counters = { - activeInfohashes: 0, - inactiveInfohashes: 0, - peers: 0, - uncompletedPeers: 0 - } - - for (const infohash of infohashes) { - const content = this.trackerServer.torrents[infohash] - - const peers = content.peers - if (peers.keys.length !== 0) counters.activeInfohashes++ - else counters.inactiveInfohashes++ - - for (const peerId of peers.keys) { - const peer = peers.peek(peerId) - if (peer == null) return - - counters.peers++ - } - } - - observableResult.observe(activeInfohashes, counters.activeInfohashes) - observableResult.observe(inactiveInfohashes, counters.inactiveInfohashes) - observableResult.observe(peers, counters.peers) - }, [ activeInfohashes, inactiveInfohashes, peers ]) - } - -} diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts deleted file mode 100644 index 47b24a54f..000000000 --- a/server/lib/opentelemetry/metric-helpers/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './bittorrent-tracker-observers-builder' -export * from './lives-observers-builder' -export * from './job-queue-observers-builder' -export * from './nodejs-observers-builder' -export * from './playback-metrics' -export * from './stats-observers-builder' -export * from './viewers-observers-builder' diff --git a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts deleted file mode 100644 index 56713ede8..000000000 --- a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Meter } from '@opentelemetry/api' -import { JobQueue } from '@server/lib/job-queue' - -export class JobQueueObserversBuilder { - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.meter.createObservableGauge('peertube_job_queue_total', { - description: 'Total jobs in the PeerTube job queue' - }).addCallback(async observableResult => { - const stats = await JobQueue.Instance.getStats() - - for (const { jobType, counts } of stats) { - for (const state of Object.keys(counts)) { - observableResult.observe(counts[state], { jobType, state }) - } - } - }) - } - -} diff --git a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts deleted file mode 100644 index 5effc18e1..000000000 --- a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Meter } from '@opentelemetry/api' -import { VideoModel } from '@server/models/video/video' - -export class LivesObserversBuilder { - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.meter.createObservableGauge('peertube_running_lives_total', { - description: 'Total running lives on the instance' - }).addCallback(async observableResult => { - const local = await VideoModel.countLives({ remote: false, mode: 'published' }) - const remote = await VideoModel.countLives({ remote: true, mode: 'published' }) - - observableResult.observe(local, { liveOrigin: 'local' }) - observableResult.observe(remote, { liveOrigin: 'remote' }) - }) - } -} diff --git a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts deleted file mode 100644 index 8ed219e9e..000000000 --- a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { readdir } from 'fs-extra' -import { constants, NodeGCPerformanceDetail, PerformanceObserver } from 'perf_hooks' -import * as process from 'process' -import { Meter, ObservableResult } from '@opentelemetry/api' -import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics' -import { View } from '@opentelemetry/sdk-metrics/build/src/view/View' -import { logger } from '@server/helpers/logger' - -// Thanks to https://github.com/siimon/prom-client -// We took their logic and adapter it for opentelemetry -// Try to keep consistency with their metric name/description so it's easier to process (grafana dashboard template etc) - -export class NodeJSObserversBuilder { - - constructor (private readonly meter: Meter) { - } - - static getViews () { - return [ - new View({ - aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]), - instrumentName: 'nodejs_gc_duration_seconds' - }) - ] - } - - buildObservers () { - this.buildCPUObserver() - this.buildMemoryObserver() - - this.buildHandlesObserver() - this.buildFileDescriptorsObserver() - - this.buildGCObserver() - this.buildEventLoopLagObserver() - - this.buildLibUVActiveRequestsObserver() - this.buildActiveResourcesObserver() - } - - private buildCPUObserver () { - const cpuTotal = this.meter.createObservableCounter('process_cpu_seconds_total', { - description: 'Total user and system CPU time spent in seconds.' - }) - const cpuUser = this.meter.createObservableCounter('process_cpu_user_seconds_total', { - description: 'Total user CPU time spent in seconds.' - }) - const cpuSystem = this.meter.createObservableCounter('process_cpu_system_seconds_total', { - description: 'Total system CPU time spent in seconds.' - }) - - let lastCpuUsage = process.cpuUsage() - - this.meter.addBatchObservableCallback(observableResult => { - const cpuUsage = process.cpuUsage() - - const userUsageMicros = cpuUsage.user - lastCpuUsage.user - const systemUsageMicros = cpuUsage.system - lastCpuUsage.system - - lastCpuUsage = cpuUsage - - observableResult.observe(cpuTotal, (userUsageMicros + systemUsageMicros) / 1e6) - observableResult.observe(cpuUser, userUsageMicros / 1e6) - observableResult.observe(cpuSystem, systemUsageMicros / 1e6) - - }, [ cpuTotal, cpuUser, cpuSystem ]) - } - - private buildMemoryObserver () { - this.meter.createObservableGauge('nodejs_memory_usage_bytes', { - description: 'Memory' - }).addCallback(observableResult => { - const current = process.memoryUsage() - - observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' }) - observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' }) - observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' }) - observableResult.observe(current.external, { memoryType: 'external' }) - observableResult.observe(current.rss, { memoryType: 'rss' }) - }) - } - - private buildHandlesObserver () { - if (typeof (process as any)._getActiveHandles !== 'function') return - - this.meter.createObservableGauge('nodejs_active_handles_total', { - description: 'Total number of active handles.' - }).addCallback(observableResult => { - const handles = (process as any)._getActiveHandles() - - observableResult.observe(handles.length) - }) - } - - private buildGCObserver () { - const kinds = { - [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major', - [constants.NODE_PERFORMANCE_GC_MINOR]: 'minor', - [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental', - [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb' - } - - const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', { - description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb' - }) - - const obs = new PerformanceObserver(list => { - const entry = list.getEntries()[0] - - // Node < 16 uses entry.kind - // Node >= 16 uses entry.detail.kind - // See: https://nodejs.org/docs/latest-v16.x/api/deprecations.html#deprecations_dep0152_extension_performanceentry_properties - const kind = entry.detail - ? kinds[(entry.detail as NodeGCPerformanceDetail).kind] - : kinds[(entry as any).kind] - - // Convert duration from milliseconds to seconds - histogram.record(entry.duration / 1000, { - kind - }) - }) - - obs.observe({ entryTypes: [ 'gc' ] }) - } - - private buildEventLoopLagObserver () { - const reportEventloopLag = (start: [ number, number ], observableResult: ObservableResult, res: () => void) => { - const delta = process.hrtime(start) - const nanosec = delta[0] * 1e9 + delta[1] - const seconds = nanosec / 1e9 - - observableResult.observe(seconds) - - res() - } - - this.meter.createObservableGauge('nodejs_eventloop_lag_seconds', { - description: 'Lag of event loop in seconds.' - }).addCallback(observableResult => { - return new Promise(res => { - const start = process.hrtime() - - setImmediate(reportEventloopLag, start, observableResult, res) - }) - }) - } - - private buildFileDescriptorsObserver () { - this.meter.createObservableGauge('process_open_fds', { - description: 'Number of open file descriptors.' - }).addCallback(async observableResult => { - try { - const fds = await readdir('/proc/self/fd') - observableResult.observe(fds.length - 1) - } catch (err) { - logger.debug('Cannot list file descriptors of current process for OpenTelemetry.', { err }) - } - }) - } - - private buildLibUVActiveRequestsObserver () { - if (typeof (process as any)._getActiveRequests !== 'function') return - - this.meter.createObservableGauge('nodejs_active_requests_total', { - description: 'Total number of active libuv requests.' - }).addCallback(observableResult => { - const requests = (process as any)._getActiveRequests() - - observableResult.observe(requests.length) - }) - } - - private buildActiveResourcesObserver () { - if (typeof (process as any).getActiveResourcesInfo !== 'function') return - - const grouped = this.meter.createObservableCounter('nodejs_active_resources', { - description: 'Number of active resources that are currently keeping the event loop alive, grouped by async resource type.' - }) - const total = this.meter.createObservableCounter('nodejs_active_resources_total', { - description: 'Total number of active resources.' - }) - - this.meter.addBatchObservableCallback(observableResult => { - const resources = (process as any).getActiveResourcesInfo() - - const data = {} - - for (let i = 0; i < resources.length; i++) { - const resource = resources[i] - - if (data[resource] === undefined) data[resource] = 0 - data[resource] += 1 - } - - for (const type of Object.keys(data)) { - observableResult.observe(grouped, data[type], { type }) - } - - observableResult.observe(total, resources.length) - }, [ grouped, total ]) - } -} diff --git a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts deleted file mode 100644 index 1eb08b5a6..000000000 --- a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { Counter, Meter } from '@opentelemetry/api' -import { MVideoImmutable } from '@server/types/models' -import { PlaybackMetricCreate } from '@shared/models' - -export class PlaybackMetrics { - private errorsCounter: Counter - private resolutionChangesCounter: Counter - - private downloadedBytesP2PCounter: Counter - private uploadedBytesP2PCounter: Counter - - private downloadedBytesHTTPCounter: Counter - - private peersP2PPeersGaugeBuffer: { - value: number - attributes: any - }[] = [] - - constructor (private readonly meter: Meter) { - - } - - buildCounters () { - this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', { - description: 'Errors collected from PeerTube player.' - }) - - this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', { - description: 'Resolution changes collected from PeerTube player.' - }) - - this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', { - description: 'Downloaded bytes with HTTP by PeerTube player.' - }) - this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', { - description: 'Downloaded bytes with P2P by PeerTube player.' - }) - - this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', { - description: 'Uploaded bytes with P2P by PeerTube player.' - }) - - this.meter.createObservableGauge('peertube_playback_p2p_peers', { - description: 'Total P2P peers connected to the PeerTube player.' - }).addCallback(observableResult => { - for (const gauge of this.peersP2PPeersGaugeBuffer) { - observableResult.observe(gauge.value, gauge.attributes) - } - - this.peersP2PPeersGaugeBuffer = [] - }) - } - - observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) { - const attributes = { - videoOrigin: video.remote - ? 'remote' - : 'local', - - playerMode: metrics.playerMode, - - resolution: metrics.resolution + '', - fps: metrics.fps + '', - - p2pEnabled: metrics.p2pEnabled, - - videoUUID: video.uuid - } - - this.errorsCounter.add(metrics.errors, attributes) - this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes) - - this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes) - this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes) - - this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes) - - if (metrics.p2pPeers) { - this.peersP2PPeersGaugeBuffer.push({ - value: metrics.p2pPeers, - attributes - }) - } - } -} diff --git a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts deleted file mode 100644 index 9f5f22e1b..000000000 --- a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts +++ /dev/null @@ -1,186 +0,0 @@ -import memoizee from 'memoizee' -import { Meter } from '@opentelemetry/api' -import { MEMOIZE_TTL } from '@server/initializers/constants' -import { buildAvailableActivities } from '@server/lib/activitypub/activity' -import { StatsManager } from '@server/lib/stat-manager' - -export class StatsObserversBuilder { - - private readonly getInstanceStats = memoizee(() => { - return StatsManager.Instance.getStats() - }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS }) - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.buildUserStatsObserver() - this.buildVideoStatsObserver() - this.buildCommentStatsObserver() - this.buildPlaylistStatsObserver() - this.buildChannelStatsObserver() - this.buildInstanceFollowsStatsObserver() - this.buildRedundancyStatsObserver() - this.buildActivityPubStatsObserver() - } - - private buildUserStatsObserver () { - this.meter.createObservableGauge('peertube_users_total', { - description: 'Total users on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalUsers) - }) - - this.meter.createObservableGauge('peertube_active_users_total', { - description: 'Total active users on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' }) - observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' }) - observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' }) - }) - } - - private buildChannelStatsObserver () { - this.meter.createObservableGauge('peertube_channels_total', { - description: 'Total channels on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' }) - }) - - this.meter.createObservableGauge('peertube_active_channels_total', { - description: 'Total active channels on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' }) - observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' }) - observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' }) - }) - } - - private buildVideoStatsObserver () { - this.meter.createObservableGauge('peertube_videos_total', { - description: 'Total videos on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' }) - observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' }) - }) - - this.meter.createObservableGauge('peertube_video_views_total', { - description: 'Total video views made on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' }) - }) - - this.meter.createObservableGauge('peertube_video_bytes_total', { - description: 'Total bytes of videos' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' }) - }) - } - - private buildCommentStatsObserver () { - this.meter.createObservableGauge('peertube_comments_total', { - description: 'Total comments on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' }) - }) - } - - private buildPlaylistStatsObserver () { - this.meter.createObservableGauge('peertube_playlists_total', { - description: 'Total playlists on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' }) - }) - } - - private buildInstanceFollowsStatsObserver () { - this.meter.createObservableGauge('peertube_instance_followers_total', { - description: 'Total followers of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalInstanceFollowers) - }) - - this.meter.createObservableGauge('peertube_instance_following_total', { - description: 'Total following of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalInstanceFollowing) - }) - } - - private buildRedundancyStatsObserver () { - this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', { - description: 'Total redundancy used of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const r of stats.videosRedundancy) { - observableResult.observe(r.totalUsed, { strategy: r.strategy }) - } - }) - - this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', { - description: 'Total redundancy available of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const r of stats.videosRedundancy) { - observableResult.observe(r.totalSize, { strategy: r.strategy }) - } - }) - } - - private buildActivityPubStatsObserver () { - const availableActivities = buildAvailableActivities() - - this.meter.createObservableGauge('peertube_ap_inbox_success_total', { - description: 'Total inbox messages processed with success' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const type of availableActivities) { - observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type }) - } - }) - - this.meter.createObservableGauge('peertube_ap_inbox_error_total', { - description: 'Total inbox messages processed with error' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const type of availableActivities) { - observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type }) - } - }) - - this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', { - description: 'Total inbox messages waiting for being processed' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalActivityPubMessagesWaiting) - }) - } -} diff --git a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts deleted file mode 100644 index c65f8ddae..000000000 --- a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Meter } from '@opentelemetry/api' -import { VideoScope, ViewerScope } from '@server/lib/views/shared' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' - -export class ViewersObserversBuilder { - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.meter.createObservableGauge('peertube_viewers_total', { - description: 'Total viewers on the instance' - }).addCallback(observableResult => { - for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) { - for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) { - const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope }) - - observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope }) - } - } - }) - } -} diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts deleted file mode 100644 index bffe00840..000000000 --- a/server/lib/opentelemetry/metrics.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Application, Request, Response } from 'express' -import { Meter, metrics } from '@opentelemetry/api' -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' -import { MeterProvider } from '@opentelemetry/sdk-metrics' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { MVideoImmutable } from '@server/types/models' -import { PlaybackMetricCreate } from '@shared/models' -import { - BittorrentTrackerObserversBuilder, - JobQueueObserversBuilder, - LivesObserversBuilder, - NodeJSObserversBuilder, - PlaybackMetrics, - StatsObserversBuilder, - ViewersObserversBuilder -} from './metric-helpers' - -class OpenTelemetryMetrics { - - private static instance: OpenTelemetryMetrics - - private meter: Meter - - private onRequestDuration: (req: Request, res: Response) => void - - private playbackMetrics: PlaybackMetrics - - private constructor () {} - - init (app: Application) { - if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return - - app.use((req, res, next) => { - res.once('finish', () => { - if (!this.onRequestDuration) return - - this.onRequestDuration(req as Request, res as Response) - }) - - next() - }) - } - - registerMetrics (options: { trackerServer: any }) { - if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return - - logger.info('Registering Open Telemetry metrics') - - const provider = new MeterProvider({ - views: [ - ...NodeJSObserversBuilder.getViews() - ] - }) - - provider.addMetricReader(new PrometheusExporter({ - host: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.HOSTNAME, - port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT - })) - - metrics.setGlobalMeterProvider(provider) - - this.meter = metrics.getMeter('default') - - if (CONFIG.OPEN_TELEMETRY.METRICS.HTTP_REQUEST_DURATION.ENABLED === true) { - this.buildRequestObserver() - } - - this.playbackMetrics = new PlaybackMetrics(this.meter) - this.playbackMetrics.buildCounters() - - const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter) - nodeJSObserversBuilder.buildObservers() - - const jobQueueObserversBuilder = new JobQueueObserversBuilder(this.meter) - jobQueueObserversBuilder.buildObservers() - - const statsObserversBuilder = new StatsObserversBuilder(this.meter) - statsObserversBuilder.buildObservers() - - const livesObserversBuilder = new LivesObserversBuilder(this.meter) - livesObserversBuilder.buildObservers() - - const viewersObserversBuilder = new ViewersObserversBuilder(this.meter) - viewersObserversBuilder.buildObservers() - - const bittorrentTrackerObserversBuilder = new BittorrentTrackerObserversBuilder(this.meter, options.trackerServer) - bittorrentTrackerObserversBuilder.buildObservers() - } - - observePlaybackMetric (video: MVideoImmutable, metrics: PlaybackMetricCreate) { - this.playbackMetrics.observe(video, metrics) - } - - private buildRequestObserver () { - const requestDuration = this.meter.createHistogram('http_request_duration_ms', { - unit: 'milliseconds', - description: 'Duration of HTTP requests in ms' - }) - - this.onRequestDuration = (req: Request, res: Response) => { - const duration = Date.now() - res.locals.requestStart - - requestDuration.record(duration, { - path: this.buildRequestPath(req.originalUrl), - method: req.method, - statusCode: res.statusCode + '' - }) - } - } - - private buildRequestPath (path: string) { - return path.split('?')[0] - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -export { - OpenTelemetryMetrics -} diff --git a/server/lib/opentelemetry/tracing.ts b/server/lib/opentelemetry/tracing.ts deleted file mode 100644 index 9a81680b2..000000000 --- a/server/lib/opentelemetry/tracing.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { SequelizeInstrumentation } from 'opentelemetry-instrumentation-sequelize' -import { context, diag, DiagLogLevel, trace } from '@opentelemetry/api' -import { JaegerExporter } from '@opentelemetry/exporter-jaeger' -import { registerInstrumentations } from '@opentelemetry/instrumentation' -import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns' -import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express' -import FsInstrumentation from '@opentelemetry/instrumentation-fs' -import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' -import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis' -import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' -import { Resource } from '@opentelemetry/resources' -import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' -import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' - -const tracer = trace.getTracer('peertube') - -function registerOpentelemetryTracing () { - if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) return - - logger.info('Registering Open Telemetry tracing') - - const customLogger = (level: string) => { - return (message: string, ...args: unknown[]) => { - let fullMessage = message - - for (const arg of args) { - if (typeof arg === 'string') fullMessage += arg - else break - } - - logger[level](fullMessage) - } - } - - diag.setLogger({ - error: customLogger('error'), - warn: customLogger('warn'), - info: customLogger('info'), - debug: customLogger('debug'), - verbose: customLogger('verbose') - }, DiagLogLevel.INFO) - - const tracerProvider = new NodeTracerProvider({ - resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: 'peertube' - }) - }) - - registerInstrumentations({ - tracerProvider, - instrumentations: [ - new PgInstrumentation({ - enhancedDatabaseReporting: true - }), - new DnsInstrumentation(), - new HttpInstrumentation(), - new ExpressInstrumentation(), - new IORedisInstrumentation({ - dbStatementSerializer: function (cmdName, cmdArgs) { - return [ cmdName, ...cmdArgs ].join(' ') - } - }), - new FsInstrumentation(), - new SequelizeInstrumentation() - ] - }) - - tracerProvider.addSpanProcessor( - new BatchSpanProcessor( - new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT }) - ) - ) - - tracerProvider.register() -} - -async function wrapWithSpanAndContext (spanName: string, cb: () => Promise) { - const span = tracer.startSpan(spanName) - const activeContext = trace.setSpan(context.active(), span) - - const result = await context.with(activeContext, () => cb()) - span.end() - - return result -} - -export { - registerOpentelemetryTracing, - tracer, - wrapWithSpanAndContext -} diff --git a/server/lib/paths.ts b/server/lib/paths.ts deleted file mode 100644 index db1cdede2..000000000 --- a/server/lib/paths.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { join } from 'path' -import { CONFIG } from '@server/initializers/config' -import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants' -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' -import { removeFragmentedMP4Ext } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { isVideoInPrivateDirectory } from './video-privacy' - -// ################## Video file name ################## - -function generateWebVideoFilename (resolution: number, extname: string) { - return buildUUID() + '-' + resolution + extname -} - -function generateHLSVideoFilename (resolution: number) { - return `${buildUUID()}-${resolution}-fragmented.mp4` -} - -// ################## Streaming playlist ################## - -function getLiveDirectory (video: MVideo) { - return getHLSDirectory(video) -} - -function getLiveReplayBaseDirectory (video: MVideo) { - return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) -} - -function getHLSDirectory (video: MVideo) { - if (isVideoInPrivateDirectory(video.privacy)) { - return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid) - } - - return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid) -} - -function getHLSRedundancyDirectory (video: MVideoUUID) { - return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) -} - -function getHlsResolutionPlaylistFilename (videoFilename: string) { - // Video file name already contain resolution - return removeFragmentedMP4Ext(videoFilename) + '.m3u8' -} - -function generateHLSMasterPlaylistFilename (isLive = false) { - if (isLive) return 'master.m3u8' - - return buildUUID() + '-master.m3u8' -} - -function generateHlsSha256SegmentsFilename (isLive = false) { - if (isLive) return 'segments-sha256.json' - - return buildUUID() + '-segments-sha256.json' -} - -// ################## Torrents ################## - -function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { - const extension = '.torrent' - const uuid = buildUUID() - - if (isStreamingPlaylist(videoOrPlaylist)) { - return `${uuid}-${resolution}-${videoOrPlaylist.getStringType()}${extension}` - } - - return uuid + '-' + resolution + extension -} - -function getFSTorrentFilePath (videoFile: MVideoFile) { - return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) -} - -// --------------------------------------------------------------------------- - -export { - generateHLSVideoFilename, - generateWebVideoFilename, - - generateTorrentFileName, - getFSTorrentFilePath, - - getHLSDirectory, - getLiveDirectory, - getLiveReplayBaseDirectory, - getHLSRedundancyDirectory, - - generateHLSMasterPlaylistFilename, - generateHlsSha256SegmentsFilename, - getHlsResolutionPlaylistFilename -} diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts deleted file mode 100644 index 3e41a2def..000000000 --- a/server/lib/peertube-socket.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { Server as HTTPServer } from 'http' -import { Namespace, Server as SocketServer, Socket } from 'socket.io' -import { isIdValid } from '@server/helpers/custom-validators/misc' -import { Debounce } from '@server/helpers/debounce' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { MRunner } from '@server/types/models/runners' -import { UserNotificationModelForApi } from '@server/types/models/user' -import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models' -import { logger } from '../helpers/logger' -import { authenticateRunnerSocket, authenticateSocket } from '../middlewares' - -class PeerTubeSocket { - - private static instance: PeerTubeSocket - - private userNotificationSockets: { [ userId: number ]: Socket[] } = {} - private liveVideosNamespace: Namespace - private readonly runnerSockets = new Set() - - private constructor () {} - - init (server: HTTPServer) { - const io = new SocketServer(server) - - io.of('/user-notifications') - .use(authenticateSocket) - .on('connection', socket => { - const userId = socket.handshake.auth.user.id - - logger.debug('User %d connected to the notification system.', userId) - - if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = [] - - this.userNotificationSockets[userId].push(socket) - - socket.on('disconnect', () => { - logger.debug('User %d disconnected from SocketIO notifications.', userId) - - this.userNotificationSockets[userId] = this.userNotificationSockets[userId].filter(s => s !== socket) - }) - }) - - this.liveVideosNamespace = io.of('/live-videos') - .on('connection', socket => { - socket.on('subscribe', ({ videoId }) => { - if (!isIdValid(videoId)) return - - /* eslint-disable @typescript-eslint/no-floating-promises */ - socket.join(videoId) - }) - - socket.on('unsubscribe', ({ videoId }) => { - if (!isIdValid(videoId)) return - - /* eslint-disable @typescript-eslint/no-floating-promises */ - socket.leave(videoId) - }) - }) - - io.of('/runners') - .use(authenticateRunnerSocket) - .on('connection', socket => { - const runner: MRunner = socket.handshake.auth.runner - - logger.debug(`New runner "${runner.name}" connected to the notification system.`) - - this.runnerSockets.add(socket) - - socket.on('disconnect', () => { - logger.debug(`Runner "${runner.name}" disconnected from the notification system.`) - - this.runnerSockets.delete(socket) - }) - }) - } - - sendNotification (userId: number, notification: UserNotificationModelForApi) { - const sockets = this.userNotificationSockets[userId] - if (!sockets) return - - logger.debug('Sending user notification to user %d.', userId) - - const notificationMessage = notification.toFormattedJSON() - for (const socket of sockets) { - socket.emit('new-notification', notificationMessage) - } - } - - sendVideoLiveNewState (video: MVideo) { - const data: LiveVideoEventPayload = { state: video.state } - const type: LiveVideoEventType = 'state-change' - - logger.debug('Sending video live new state notification of %s.', video.url, { state: video.state }) - - this.liveVideosNamespace - .in(video.id) - .emit(type, data) - } - - sendVideoViewsUpdate (video: MVideoImmutable, numViewers: number) { - const data: LiveVideoEventPayload = { viewers: numViewers } - const type: LiveVideoEventType = 'views-change' - - logger.debug('Sending video live views update notification of %s.', video.url, { viewers: numViewers }) - - this.liveVideosNamespace - .in(video.id) - .emit(type, data) - } - - @Debounce({ timeoutMS: 1000 }) - sendAvailableJobsPingToRunners () { - logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`) - - for (const runners of this.runnerSockets) { - runners.emit('available-jobs') - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - PeerTubeSocket -} diff --git a/server/lib/plugins/hooks.ts b/server/lib/plugins/hooks.ts deleted file mode 100644 index 694527c12..000000000 --- a/server/lib/plugins/hooks.ts +++ /dev/null @@ -1,35 +0,0 @@ -import Bluebird from 'bluebird' -import { ServerActionHookName, ServerFilterHookName } from '../../../shared/models' -import { logger } from '../../helpers/logger' -import { PluginManager } from './plugin-manager' - -type PromiseFunction = (params: U) => Promise | Bluebird -type RawFunction = (params: U) => T - -// Helpers to run hooks -const Hooks = { - wrapObject: (result: T, hookName: U, context?: any) => { - return PluginManager.Instance.runHook(hookName, result, context) - }, - - wrapPromiseFun: async (fun: PromiseFunction, params: U, hookName: V) => { - const result = await fun(params) - - return PluginManager.Instance.runHook(hookName, result, params) - }, - - wrapFun: async (fun: RawFunction, params: U, hookName: V) => { - const result = fun(params) - - return PluginManager.Instance.runHook(hookName, result, params) - }, - - runAction: (hookName: U, params?: T) => { - PluginManager.Instance.runHook(hookName, undefined, params) - .catch(err => logger.error('Fatal hook error.', { err })) - } -} - -export { - Hooks -} diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts deleted file mode 100644 index b4e3eece4..000000000 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ /dev/null @@ -1,262 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { join } from 'path' -import { buildLogger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { AccountModel } from '@server/models/account/account' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { getServerActor } from '@server/models/application/application' -import { ServerModel } from '@server/models/server/server' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoBlacklistModel } from '@server/models/video/video-blacklist' -import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models' -import { PeerTubeHelpers } from '@server/types/plugins' -import { ffprobePromise } from '@shared/ffmpeg' -import { VideoBlacklistCreate, VideoStorage } from '@shared/models' -import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' -import { PeerTubeSocket } from '../peertube-socket' -import { ServerConfigManager } from '../server-config-manager' -import { blacklistVideo, unblacklistVideo } from '../video-blacklist' -import { VideoPathManager } from '../video-path-manager' - -function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers { - const logger = buildPluginLogger(npmName) - - const database = buildDatabaseHelpers() - const videos = buildVideosHelpers() - - const config = buildConfigHelpers() - - const server = buildServerHelpers(httpServer) - - const moderation = buildModerationHelpers() - - const plugin = buildPluginRelatedHelpers(pluginModel, npmName) - - const socket = buildSocketHelpers() - - const user = buildUserHelpers() - - return { - logger, - database, - videos, - config, - moderation, - plugin, - server, - socket, - user - } -} - -export { - buildPluginHelpers -} - -// --------------------------------------------------------------------------- - -function buildPluginLogger (npmName: string) { - return buildLogger(npmName) -} - -function buildDatabaseHelpers () { - return { - query: sequelizeTypescript.query.bind(sequelizeTypescript) - } -} - -function buildServerHelpers (httpServer: Server) { - return { - getHTTPServer: () => httpServer, - - getServerActor: () => getServerActor() - } -} - -function buildVideosHelpers () { - return { - loadByUrl: (url: string) => { - return VideoModel.loadByUrl(url) - }, - - loadByIdOrUUID: (id: number | string) => { - return VideoModel.load(id) - }, - - removeVideo: (id: number) => { - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(id, t) - - await video.destroy({ transaction: t }) - }) - }, - - ffprobe: (path: string) => { - return ffprobePromise(path) - }, - - getFiles: async (id: number | string) => { - const video = await VideoModel.loadFull(id) - if (!video) return undefined - - const webVideoFiles = (video.VideoFiles || []).map(f => ({ - path: f.storage === VideoStorage.FILE_SYSTEM - ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f) - : null, - url: f.getFileUrl(video), - - resolution: f.resolution, - size: f.size, - fps: f.fps - })) - - const hls = video.getHLSPlaylist() - - const hlsVideoFiles = hls - ? (video.getHLSPlaylist().VideoFiles || []).map(f => { - return { - path: f.storage === VideoStorage.FILE_SYSTEM - ? VideoPathManager.Instance.getFSVideoFileOutputPath(hls, f) - : null, - url: f.getFileUrl(video), - resolution: f.resolution, - size: f.size, - fps: f.fps - } - }) - : [] - - const thumbnails = video.Thumbnails.map(t => ({ - type: t.type, - url: t.getOriginFileUrl(video), - path: t.getPath() - })) - - return { - webtorrent: { // TODO: remove in v7 - videoFiles: webVideoFiles - }, - - webVideo: { - videoFiles: webVideoFiles - }, - - hls: { - videoFiles: hlsVideoFiles - }, - - thumbnails - } - } - } -} - -function buildModerationHelpers () { - return { - blockServer: async (options: { byAccountId: number, hostToBlock: string }) => { - const serverToBlock = await ServerModel.loadOrCreateByHost(options.hostToBlock) - - await addServerInBlocklist(options.byAccountId, serverToBlock.id) - }, - - unblockServer: async (options: { byAccountId: number, hostToUnblock: string }) => { - const serverBlock = await ServerBlocklistModel.loadByAccountAndHost(options.byAccountId, options.hostToUnblock) - if (!serverBlock) return - - await removeServerFromBlocklist(serverBlock) - }, - - blockAccount: async (options: { byAccountId: number, handleToBlock: string }) => { - const accountToBlock = await AccountModel.loadByNameWithHost(options.handleToBlock) - if (!accountToBlock) return - - await addAccountInBlocklist(options.byAccountId, accountToBlock.id) - }, - - unblockAccount: async (options: { byAccountId: number, handleToUnblock: string }) => { - const targetAccount = await AccountModel.loadByNameWithHost(options.handleToUnblock) - if (!targetAccount) return - - const accountBlock = await AccountBlocklistModel.loadByAccountAndTarget(options.byAccountId, targetAccount.id) - if (!accountBlock) return - - await removeAccountFromBlocklist(accountBlock) - }, - - blacklistVideo: async (options: { videoIdOrUUID: number | string, createOptions: VideoBlacklistCreate }) => { - const video = await VideoModel.loadFull(options.videoIdOrUUID) - if (!video) return - - await blacklistVideo(video, options.createOptions) - }, - - unblacklistVideo: async (options: { videoIdOrUUID: number | string }) => { - const video = await VideoModel.loadFull(options.videoIdOrUUID) - if (!video) return - - const videoBlacklist = await VideoBlacklistModel.loadByVideoId(video.id) - if (!videoBlacklist) return - - await unblacklistVideo(videoBlacklist, video) - } - } -} - -function buildConfigHelpers () { - return { - getWebserverUrl () { - return WEBSERVER.URL - }, - - getServerListeningConfig () { - return { hostname: CONFIG.LISTEN.HOSTNAME, port: CONFIG.LISTEN.PORT } - }, - - getServerConfig () { - return ServerConfigManager.Instance.getServerConfig() - } - } -} - -function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) { - return { - getBaseStaticRoute: () => `/plugins/${plugin.name}/${plugin.version}/static/`, - - getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, - - getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`, - - getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) - } -} - -function buildSocketHelpers () { - return { - sendNotification: (userId: number, notification: UserNotificationModelForApi) => { - PeerTubeSocket.Instance.sendNotification(userId, notification) - }, - sendVideoLiveNewState: (video: MVideo) => { - PeerTubeSocket.Instance.sendVideoLiveNewState(video) - } - } -} - -function buildUserHelpers () { - return { - loadById: (id: number) => { - return UserModel.loadByIdFull(id) - }, - - getAuthUser: (res: express.Response) => { - const user = res.locals.oauth?.token?.User || res.locals.videoFileToken?.user - if (!user) return undefined - - return UserModel.loadByIdFull(user.id) - } - } -} diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts deleted file mode 100644 index 119cee8e0..000000000 --- a/server/lib/plugins/plugin-index.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { sanitizeUrl } from '@server/helpers/core-utils' -import { logger } from '@server/helpers/logger' -import { doJSONRequest } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { PEERTUBE_VERSION } from '@server/initializers/constants' -import { PluginModel } from '@server/models/server/plugin' -import { - PeerTubePluginIndex, - PeertubePluginIndexList, - PeertubePluginLatestVersionRequest, - PeertubePluginLatestVersionResponse, - ResultList -} from '@shared/models' -import { PluginManager } from './plugin-manager' - -async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { - const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options - - const searchParams: PeertubePluginIndexList & Record = { - start, - count, - sort, - pluginType, - search, - currentPeerTubeEngine: options.currentPeerTubeEngine || PEERTUBE_VERSION - } - - const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' - - try { - const { body } = await doJSONRequest(uri, { searchParams }) - - logger.debug('Got result from PeerTube index.', { body }) - - addInstanceInformation(body) - - return body as ResultList - } catch (err) { - logger.error('Cannot list available plugins from index %s.', uri, { err }) - return undefined - } -} - -function addInstanceInformation (result: ResultList) { - for (const d of result.data) { - d.installed = PluginManager.Instance.isRegistered(d.npmName) - d.name = PluginModel.normalizePluginName(d.npmName) - } - - return result -} - -async function getLatestPluginsVersion (npmNames: string[]): Promise { - const bodyRequest: PeertubePluginLatestVersionRequest = { - npmNames, - currentPeerTubeEngine: PEERTUBE_VERSION - } - - const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version' - - const options = { - json: bodyRequest, - method: 'POST' as 'POST' - } - const { body } = await doJSONRequest(uri, options) - - return body -} - -async function getLatestPluginVersion (npmName: string) { - const results = await getLatestPluginsVersion([ npmName ]) - - if (Array.isArray(results) === false || results.length !== 1) { - logger.warn('Cannot get latest supported plugin version of %s.', npmName) - return undefined - } - - return results[0].latestVersion -} - -export { - listAvailablePluginsFromIndex, - getLatestPluginVersion, - getLatestPluginsVersion -} diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts deleted file mode 100644 index 88c5b60d7..000000000 --- a/server/lib/plugins/plugin-manager.ts +++ /dev/null @@ -1,665 +0,0 @@ -import express from 'express' -import { createReadStream, createWriteStream } from 'fs' -import { ensureDir, outputFile, readJSON } from 'fs-extra' -import { Server } from 'http' -import { basename, join } from 'path' -import { decachePlugin } from '@server/helpers/decache' -import { ApplicationModel } from '@server/models/application/application' -import { MOAuthTokenUser, MUser } from '@server/types/models' -import { getCompleteLocale } from '@shared/core-utils' -import { - ClientScriptJSON, - PluginPackageJSON, - PluginTranslation, - PluginTranslationPathsJSON, - RegisterServerHookOptions -} from '@shared/models' -import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' -import { PluginType } from '../../../shared/models/plugins/plugin.type' -import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' -import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { PLUGIN_GLOBAL_CSS_PATH } from '../../initializers/constants' -import { PluginModel } from '../../models/server/plugin' -import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPassOptions, RegisterServerOptions } from '../../types/plugins' -import { ClientHtml } from '../client-html' -import { RegisterHelpers } from './register-helpers' -import { installNpmPlugin, installNpmPluginFromDisk, rebuildNativePlugins, removeNpmPlugin } from './yarn' - -export interface RegisteredPlugin { - npmName: string - name: string - version: string - description: string - peertubeEngine: string - - type: PluginType - - path: string - - staticDirs: { [name: string]: string } - clientScripts: { [name: string]: ClientScriptJSON } - - css: string[] - - // Only if this is a plugin - registerHelpers?: RegisterHelpers - unregister?: Function -} - -export interface HookInformationValue { - npmName: string - pluginName: string - handler: Function - priority: number -} - -type PluginLocalesTranslations = { - [locale: string]: PluginTranslation -} - -export class PluginManager implements ServerHook { - - private static instance: PluginManager - - private registeredPlugins: { [name: string]: RegisteredPlugin } = {} - - private hooks: { [name: string]: HookInformationValue[] } = {} - private translations: PluginLocalesTranslations = {} - - private server: Server - - private constructor () { - } - - init (server: Server) { - this.server = server - } - - registerWebSocketRouter () { - this.server.on('upgrade', (request, socket, head) => { - // Check if it's a plugin websocket connection - // No need to destroy the stream when we abort the request - // Other handlers in PeerTube will catch this upgrade event too (socket.io, tracker etc) - - const url = request.url - - const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`) - if (!matched) return - - const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN) - const subRoute = matched[3] - - const result = this.getRegisteredPluginOrTheme(npmName) - if (!result) return - - const routes = result.registerHelpers.getWebSocketRoutes() - - const wss = routes.find(r => r.route.startsWith(subRoute)) - if (!wss) return - - try { - wss.handler(request, socket, head) - } catch (err) { - logger.error('Exception in plugin handler ' + npmName, { err }) - } - }) - } - - // ###################### Getters ###################### - - isRegistered (npmName: string) { - return !!this.getRegisteredPluginOrTheme(npmName) - } - - getRegisteredPluginOrTheme (npmName: string) { - return this.registeredPlugins[npmName] - } - - getRegisteredPluginByShortName (name: string) { - const npmName = PluginModel.buildNpmName(name, PluginType.PLUGIN) - const registered = this.getRegisteredPluginOrTheme(npmName) - - if (!registered || registered.type !== PluginType.PLUGIN) return undefined - - return registered - } - - getRegisteredThemeByShortName (name: string) { - const npmName = PluginModel.buildNpmName(name, PluginType.THEME) - const registered = this.getRegisteredPluginOrTheme(npmName) - - if (!registered || registered.type !== PluginType.THEME) return undefined - - return registered - } - - getRegisteredPlugins () { - return this.getRegisteredPluginsOrThemes(PluginType.PLUGIN) - } - - getRegisteredThemes () { - return this.getRegisteredPluginsOrThemes(PluginType.THEME) - } - - getIdAndPassAuths () { - return this.getRegisteredPlugins() - .map(p => ({ - npmName: p.npmName, - name: p.name, - version: p.version, - idAndPassAuths: p.registerHelpers.getIdAndPassAuths() - })) - .filter(v => v.idAndPassAuths.length !== 0) - } - - getExternalAuths () { - return this.getRegisteredPlugins() - .map(p => ({ - npmName: p.npmName, - name: p.name, - version: p.version, - externalAuths: p.registerHelpers.getExternalAuths() - })) - .filter(v => v.externalAuths.length !== 0) - } - - getRegisteredSettings (npmName: string) { - const result = this.getRegisteredPluginOrTheme(npmName) - if (!result || result.type !== PluginType.PLUGIN) return [] - - return result.registerHelpers.getSettings() - } - - getRouter (npmName: string) { - const result = this.getRegisteredPluginOrTheme(npmName) - if (!result || result.type !== PluginType.PLUGIN) return null - - return result.registerHelpers.getRouter() - } - - getTranslations (locale: string) { - return this.translations[locale] || {} - } - - async isTokenValid (token: MOAuthTokenUser, type: 'access' | 'refresh') { - const auth = this.getAuth(token.User.pluginAuth, token.authName) - if (!auth) return true - - if (auth.hookTokenValidity) { - try { - const { valid } = await auth.hookTokenValidity({ token, type }) - - if (valid === false) { - logger.info('Rejecting %s token validity from auth %s of plugin %s', type, token.authName, token.User.pluginAuth) - } - - return valid - } catch (err) { - logger.warn('Cannot run check token validity from auth %s of plugin %s.', token.authName, token.User.pluginAuth, { err }) - return true - } - } - - return true - } - - // ###################### External events ###################### - - async onLogout (npmName: string, authName: string, user: MUser, req: express.Request) { - const auth = this.getAuth(npmName, authName) - - if (auth?.onLogout) { - logger.info('Running onLogout function from auth %s of plugin %s', authName, npmName) - - try { - // Force await, in case or onLogout returns a promise - const result = await auth.onLogout(user, req) - - return typeof result === 'string' - ? result - : undefined - } catch (err) { - logger.warn('Cannot run onLogout function from auth %s of plugin %s.', authName, npmName, { err }) - } - } - - return undefined - } - - async onSettingsChanged (name: string, settings: any) { - const registered = this.getRegisteredPluginByShortName(name) - if (!registered) { - logger.error('Cannot find plugin %s to call on settings changed.', name) - } - - for (const cb of registered.registerHelpers.getOnSettingsChangedCallbacks()) { - try { - await cb(settings) - } catch (err) { - logger.error('Cannot run on settings changed callback for %s.', registered.npmName, { err }) - } - } - } - - // ###################### Hooks ###################### - - async runHook (hookName: ServerHookName, result?: T, params?: any): Promise { - if (!this.hooks[hookName]) return Promise.resolve(result) - - const hookType = getHookType(hookName) - - for (const hook of this.hooks[hookName]) { - logger.debug('Running hook %s of plugin %s.', hookName, hook.npmName) - - result = await internalRunHook({ - handler: hook.handler, - hookType, - result, - params, - onError: err => { logger.error('Cannot run hook %s of plugin %s.', hookName, hook.pluginName, { err }) } - }) - } - - return result - } - - // ###################### Registration ###################### - - async registerPluginsAndThemes () { - await this.resetCSSGlobalFile() - - const plugins = await PluginModel.listEnabledPluginsAndThemes() - - for (const plugin of plugins) { - try { - await this.registerPluginOrTheme(plugin) - } catch (err) { - // Try to unregister the plugin - try { - await this.unregister(PluginModel.buildNpmName(plugin.name, plugin.type)) - } catch { - // we don't care if we cannot unregister it - } - - logger.error('Cannot register plugin %s, skipping.', plugin.name, { err }) - } - } - - this.sortHooksByPriority() - } - - // Don't need the plugin type since themes cannot register server code - async unregister (npmName: string) { - logger.info('Unregister plugin %s.', npmName) - - const plugin = this.getRegisteredPluginOrTheme(npmName) - - if (!plugin) { - throw new Error(`Unknown plugin ${npmName} to unregister`) - } - - delete this.registeredPlugins[plugin.npmName] - - this.deleteTranslations(plugin.npmName) - - if (plugin.type === PluginType.PLUGIN) { - await plugin.unregister() - - // Remove hooks of this plugin - for (const key of Object.keys(this.hooks)) { - this.hooks[key] = this.hooks[key].filter(h => h.npmName !== npmName) - } - - const store = plugin.registerHelpers - store.reinitVideoConstants(plugin.npmName) - store.reinitTranscodingProfilesAndEncoders(plugin.npmName) - - logger.info('Regenerating registered plugin CSS to global file.') - await this.regeneratePluginGlobalCSS() - } - - ClientHtml.invalidCache() - } - - // ###################### Installation ###################### - - async install (options: { - toInstall: string - version?: string - fromDisk?: boolean // default false - register?: boolean // default true - }) { - const { toInstall, version, fromDisk = false, register = true } = options - - let plugin: PluginModel - let npmName: string - - logger.info('Installing plugin %s.', toInstall) - - try { - fromDisk - ? await installNpmPluginFromDisk(toInstall) - : await installNpmPlugin(toInstall, version) - - npmName = fromDisk ? basename(toInstall) : toInstall - const pluginType = PluginModel.getTypeFromNpmName(npmName) - const pluginName = PluginModel.normalizePluginName(npmName) - - const packageJSON = await this.getPackageJSON(pluginName, pluginType) - - this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, pluginType); - - [ plugin ] = await PluginModel.upsert({ - name: pluginName, - description: packageJSON.description, - homepage: packageJSON.homepage, - type: pluginType, - version: packageJSON.version, - enabled: true, - uninstalled: false, - peertubeEngine: packageJSON.engine.peertube - }, { returning: true }) - - logger.info('Successful installation of plugin %s.', toInstall) - - if (register) { - await this.registerPluginOrTheme(plugin) - } - } catch (rootErr) { - logger.error('Cannot install plugin %s, removing it...', toInstall, { err: rootErr }) - - if (npmName) { - try { - await this.uninstall({ npmName }) - } catch (err) { - logger.error('Cannot uninstall plugin %s after failed installation.', toInstall, { err }) - - try { - await removeNpmPlugin(npmName) - } catch (err) { - logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) - } - } - } - - throw rootErr - } - - return plugin - } - - async update (toUpdate: string, fromDisk = false) { - const npmName = fromDisk ? basename(toUpdate) : toUpdate - - logger.info('Updating plugin %s.', npmName) - - // Use the latest version from DB, to not upgrade to a version that does not support our PeerTube version - let version: string - if (!fromDisk) { - const plugin = await PluginModel.loadByNpmName(toUpdate) - version = plugin.latestVersion - } - - // Unregister old hooks - await this.unregister(npmName) - - return this.install({ toInstall: toUpdate, version, fromDisk }) - } - - async uninstall (options: { - npmName: string - unregister?: boolean // default true - }) { - const { npmName, unregister = true } = options - - logger.info('Uninstalling plugin %s.', npmName) - - if (unregister) { - try { - await this.unregister(npmName) - } catch (err) { - logger.warn('Cannot unregister plugin %s.', npmName, { err }) - } - } - - const plugin = await PluginModel.loadByNpmName(npmName) - if (!plugin || plugin.uninstalled === true) { - logger.error('Cannot uninstall plugin %s: it does not exist or is already uninstalled.', npmName) - return - } - - plugin.enabled = false - plugin.uninstalled = true - - await plugin.save() - - await removeNpmPlugin(npmName) - - logger.info('Plugin %s uninstalled.', npmName) - } - - async rebuildNativePluginsIfNeeded () { - if (!await ApplicationModel.nodeABIChanged()) return - - return rebuildNativePlugins() - } - - // ###################### Private register ###################### - - private async registerPluginOrTheme (plugin: PluginModel) { - const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) - - logger.info('Registering plugin or theme %s.', npmName) - - const packageJSON = await this.getPackageJSON(plugin.name, plugin.type) - const pluginPath = this.getPluginPath(plugin.name, plugin.type) - - this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type) - - let library: PluginLibrary - let registerHelpers: RegisterHelpers - if (plugin.type === PluginType.PLUGIN) { - const result = await this.registerPlugin(plugin, pluginPath, packageJSON) - library = result.library - registerHelpers = result.registerStore - } - - const clientScripts: { [id: string]: ClientScriptJSON } = {} - for (const c of packageJSON.clientScripts) { - clientScripts[c.script] = c - } - - this.registeredPlugins[npmName] = { - npmName, - name: plugin.name, - type: plugin.type, - version: plugin.version, - description: plugin.description, - peertubeEngine: plugin.peertubeEngine, - path: pluginPath, - staticDirs: packageJSON.staticDirs, - clientScripts, - css: packageJSON.css, - registerHelpers: registerHelpers || undefined, - unregister: library ? library.unregister : undefined - } - - await this.addTranslations(plugin, npmName, packageJSON.translations) - - ClientHtml.invalidCache() - } - - private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) { - const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) - - // Delete cache if needed - const modulePath = join(pluginPath, packageJSON.library) - decachePlugin(modulePath) - const library: PluginLibrary = require(modulePath) - - if (!isLibraryCodeValid(library)) { - throw new Error('Library code is not valid (miss register or unregister function)') - } - - const { registerOptions, registerStore } = this.getRegisterHelpers(npmName, plugin) - - await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath()) - - await library.register(registerOptions) - - logger.info('Add plugin %s CSS to global file.', npmName) - - await this.addCSSToGlobalFile(pluginPath, packageJSON.css) - - return { library, registerStore } - } - - // ###################### Translations ###################### - - private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) { - for (const locale of Object.keys(translationPaths)) { - const path = translationPaths[locale] - const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) - - const completeLocale = getCompleteLocale(locale) - - if (!this.translations[completeLocale]) this.translations[completeLocale] = {} - this.translations[completeLocale][npmName] = json - - logger.info('Added locale %s of plugin %s.', completeLocale, npmName) - } - } - - private deleteTranslations (npmName: string) { - for (const locale of Object.keys(this.translations)) { - delete this.translations[locale][npmName] - - logger.info('Deleted locale %s of plugin %s.', locale, npmName) - } - } - - // ###################### CSS ###################### - - private resetCSSGlobalFile () { - return outputFile(PLUGIN_GLOBAL_CSS_PATH, '') - } - - private async addCSSToGlobalFile (pluginPath: string, cssRelativePaths: string[]) { - for (const cssPath of cssRelativePaths) { - await this.concatFiles(join(pluginPath, cssPath), PLUGIN_GLOBAL_CSS_PATH) - } - } - - private concatFiles (input: string, output: string) { - return new Promise((res, rej) => { - const inputStream = createReadStream(input) - const outputStream = createWriteStream(output, { flags: 'a' }) - - inputStream.pipe(outputStream) - - inputStream.on('end', () => res()) - inputStream.on('error', err => rej(err)) - }) - } - - private async regeneratePluginGlobalCSS () { - await this.resetCSSGlobalFile() - - for (const plugin of this.getRegisteredPlugins()) { - await this.addCSSToGlobalFile(plugin.path, plugin.css) - } - } - - // ###################### Utils ###################### - - private sortHooksByPriority () { - for (const hookName of Object.keys(this.hooks)) { - this.hooks[hookName].sort((a, b) => { - return b.priority - a.priority - }) - } - } - - private getPackageJSON (pluginName: string, pluginType: PluginType) { - const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json') - - return readJSON(pluginPath) as Promise - } - - private getPluginPath (pluginName: string, pluginType: PluginType) { - const npmName = PluginModel.buildNpmName(pluginName, pluginType) - - return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', npmName) - } - - private getAuth (npmName: string, authName: string) { - const plugin = this.getRegisteredPluginOrTheme(npmName) - if (!plugin || plugin.type !== PluginType.PLUGIN) return null - - let auths: (RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions)[] = plugin.registerHelpers.getIdAndPassAuths() - auths = auths.concat(plugin.registerHelpers.getExternalAuths()) - - return auths.find(a => a.authName === authName) - } - - // ###################### Private getters ###################### - - private getRegisteredPluginsOrThemes (type: PluginType) { - const plugins: RegisteredPlugin[] = [] - - for (const npmName of Object.keys(this.registeredPlugins)) { - const plugin = this.registeredPlugins[npmName] - if (plugin.type !== type) continue - - plugins.push(plugin) - } - - return plugins - } - - // ###################### Generate register helpers ###################### - - private getRegisterHelpers ( - npmName: string, - plugin: PluginModel - ): { registerStore: RegisterHelpers, registerOptions: RegisterServerOptions } { - const onHookAdded = (options: RegisterServerHookOptions) => { - if (!this.hooks[options.target]) this.hooks[options.target] = [] - - this.hooks[options.target].push({ - npmName, - pluginName: plugin.name, - handler: options.handler, - priority: options.priority || 0 - }) - } - - const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this)) - - return { - registerStore: registerHelpers, - registerOptions: registerHelpers.buildRegisterHelpers() - } - } - - private sanitizeAndCheckPackageJSONOrThrow (packageJSON: PluginPackageJSON, pluginType: PluginType) { - if (!packageJSON.staticDirs) packageJSON.staticDirs = {} - if (!packageJSON.css) packageJSON.css = [] - if (!packageJSON.clientScripts) packageJSON.clientScripts = [] - if (!packageJSON.translations) packageJSON.translations = {} - - const { result: packageJSONValid, badFields } = isPackageJSONValid(packageJSON, pluginType) - if (!packageJSONValid) { - const formattedFields = badFields.map(f => `"${f}"`) - .join(', ') - - throw new Error(`PackageJSON is invalid (invalid fields: ${formattedFields}).`) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts deleted file mode 100644 index 1aaef3606..000000000 --- a/server/lib/plugins/register-helpers.ts +++ /dev/null @@ -1,340 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { logger } from '@server/helpers/logger' -import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' -import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' -import { PluginModel } from '@server/models/server/plugin' -import { - RegisterServerAuthExternalOptions, - RegisterServerAuthExternalResult, - RegisterServerAuthPassOptions, - RegisterServerExternalAuthenticatedResult, - RegisterServerOptions, - RegisterServerWebSocketRouteOptions -} from '@server/types/plugins' -import { - EncoderOptionsBuilder, - PluginSettingsManager, - PluginStorageManager, - RegisterServerHookOptions, - RegisterServerSettingOptions, - serverHookObject, - SettingsChangeCallback, - VideoPlaylistPrivacy, - VideoPrivacy -} from '@shared/models' -import { VideoTranscodingProfilesManager } from '../transcoding/default-transcoding-profiles' -import { buildPluginHelpers } from './plugin-helpers-builder' - -export class RegisterHelpers { - private readonly transcodingProfiles: { - [ npmName: string ]: { - type: 'vod' | 'live' - encoder: string - profile: string - }[] - } = {} - - private readonly transcodingEncoders: { - [ npmName: string ]: { - type: 'vod' | 'live' - streamType: 'audio' | 'video' - encoder: string - priority: number - }[] - } = {} - - private readonly settings: RegisterServerSettingOptions[] = [] - - private idAndPassAuths: RegisterServerAuthPassOptions[] = [] - private externalAuths: RegisterServerAuthExternalOptions[] = [] - - private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] - - private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = [] - - private readonly router: express.Router - private readonly videoConstantManagerFactory: VideoConstantManagerFactory - - constructor ( - private readonly npmName: string, - private readonly plugin: PluginModel, - private readonly server: Server, - private readonly onHookAdded: (options: RegisterServerHookOptions) => void - ) { - this.router = express.Router() - this.videoConstantManagerFactory = new VideoConstantManagerFactory(this.npmName) - } - - buildRegisterHelpers (): RegisterServerOptions { - const registerHook = this.buildRegisterHook() - const registerSetting = this.buildRegisterSetting() - - const getRouter = this.buildGetRouter() - const registerWebSocketRoute = this.buildRegisterWebSocketRoute() - - const settingsManager = this.buildSettingsManager() - const storageManager = this.buildStorageManager() - - const videoLanguageManager = this.videoConstantManagerFactory.createVideoConstantManager('language') - - const videoLicenceManager = this.videoConstantManagerFactory.createVideoConstantManager('licence') - const videoCategoryManager = this.videoConstantManagerFactory.createVideoConstantManager('category') - - const videoPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager('privacy') - const playlistPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager('playlistPrivacy') - - const transcodingManager = this.buildTranscodingManager() - - const registerIdAndPassAuth = this.buildRegisterIdAndPassAuth() - const registerExternalAuth = this.buildRegisterExternalAuth() - const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() - const unregisterExternalAuth = this.buildUnregisterExternalAuth() - - const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName) - - return { - registerHook, - registerSetting, - - getRouter, - registerWebSocketRoute, - - settingsManager, - storageManager, - - videoLanguageManager: { - ...videoLanguageManager, - /** @deprecated use `addConstant` instead **/ - addLanguage: videoLanguageManager.addConstant, - /** @deprecated use `deleteConstant` instead **/ - deleteLanguage: videoLanguageManager.deleteConstant - }, - videoCategoryManager: { - ...videoCategoryManager, - /** @deprecated use `addConstant` instead **/ - addCategory: videoCategoryManager.addConstant, - /** @deprecated use `deleteConstant` instead **/ - deleteCategory: videoCategoryManager.deleteConstant - }, - videoLicenceManager: { - ...videoLicenceManager, - /** @deprecated use `addConstant` instead **/ - addLicence: videoLicenceManager.addConstant, - /** @deprecated use `deleteConstant` instead **/ - deleteLicence: videoLicenceManager.deleteConstant - }, - - videoPrivacyManager: { - ...videoPrivacyManager, - /** @deprecated use `deleteConstant` instead **/ - deletePrivacy: videoPrivacyManager.deleteConstant - }, - playlistPrivacyManager: { - ...playlistPrivacyManager, - /** @deprecated use `deleteConstant` instead **/ - deletePlaylistPrivacy: playlistPrivacyManager.deleteConstant - }, - - transcodingManager, - - registerIdAndPassAuth, - registerExternalAuth, - unregisterIdAndPassAuth, - unregisterExternalAuth, - - peertubeHelpers - } - } - - reinitVideoConstants (npmName: string) { - this.videoConstantManagerFactory.resetVideoConstants(npmName) - } - - reinitTranscodingProfilesAndEncoders (npmName: string) { - const profiles = this.transcodingProfiles[npmName] - if (Array.isArray(profiles)) { - for (const profile of profiles) { - VideoTranscodingProfilesManager.Instance.removeProfile(profile) - } - } - - const encoders = this.transcodingEncoders[npmName] - if (Array.isArray(encoders)) { - for (const o of encoders) { - VideoTranscodingProfilesManager.Instance.removeEncoderPriority(o.type, o.streamType, o.encoder, o.priority) - } - } - } - - getSettings () { - return this.settings - } - - getRouter () { - return this.router - } - - getIdAndPassAuths () { - return this.idAndPassAuths - } - - getExternalAuths () { - return this.externalAuths - } - - getOnSettingsChangedCallbacks () { - return this.onSettingsChangeCallbacks - } - - getWebSocketRoutes () { - return this.webSocketRoutes - } - - private buildGetRouter () { - return () => this.router - } - - private buildRegisterWebSocketRoute () { - return (options: RegisterServerWebSocketRouteOptions) => { - this.webSocketRoutes.push(options) - } - } - - private buildRegisterSetting () { - return (options: RegisterServerSettingOptions) => { - this.settings.push(options) - } - } - - private buildRegisterHook () { - return (options: RegisterServerHookOptions) => { - if (serverHookObject[options.target] !== true) { - logger.warn('Unknown hook %s of plugin %s. Skipping.', options.target, this.npmName) - return - } - - return this.onHookAdded(options) - } - } - - private buildRegisterIdAndPassAuth () { - return (options: RegisterServerAuthPassOptions) => { - if (!options.authName || typeof options.getWeight !== 'function' || typeof options.login !== 'function') { - logger.error('Cannot register auth plugin %s: authName, getWeight or login are not valid.', this.npmName, { options }) - return - } - - this.idAndPassAuths.push(options) - } - } - - private buildRegisterExternalAuth () { - const self = this - - return (options: RegisterServerAuthExternalOptions) => { - if (!options.authName || typeof options.authDisplayName !== 'function' || typeof options.onAuthRequest !== 'function') { - logger.error('Cannot register auth plugin %s: authName, authDisplayName or onAuthRequest are not valid.', this.npmName, { options }) - return - } - - this.externalAuths.push(options) - - return { - userAuthenticated (result: RegisterServerExternalAuthenticatedResult): void { - onExternalUserAuthenticated({ - npmName: self.npmName, - authName: options.authName, - authResult: result - }).catch(err => { - logger.error('Cannot execute onExternalUserAuthenticated.', { npmName: self.npmName, authName: options.authName, err }) - }) - } - } as RegisterServerAuthExternalResult - } - } - - private buildUnregisterExternalAuth () { - return (authName: string) => { - this.externalAuths = this.externalAuths.filter(a => a.authName !== authName) - } - } - - private buildUnregisterIdAndPassAuth () { - return (authName: string) => { - this.idAndPassAuths = this.idAndPassAuths.filter(a => a.authName !== authName) - } - } - - private buildSettingsManager (): PluginSettingsManager { - return { - getSetting: (name: string) => PluginModel.getSetting(this.plugin.name, this.plugin.type, name, this.settings), - - getSettings: (names: string[]) => PluginModel.getSettings(this.plugin.name, this.plugin.type, names, this.settings), - - setSetting: (name: string, value: string) => PluginModel.setSetting(this.plugin.name, this.plugin.type, name, value), - - onSettingsChange: (cb: SettingsChangeCallback) => this.onSettingsChangeCallbacks.push(cb) - } - } - - private buildStorageManager (): PluginStorageManager { - return { - getData: (key: string) => PluginModel.getData(this.plugin.name, this.plugin.type, key), - - storeData: (key: string, data: any) => PluginModel.storeData(this.plugin.name, this.plugin.type, key, data) - } - } - - private buildTranscodingManager () { - const self = this - - function addProfile (type: 'live' | 'vod', encoder: string, profile: string, builder: EncoderOptionsBuilder) { - if (profile === 'default') { - logger.error('A plugin cannot add a default live transcoding profile') - return false - } - - VideoTranscodingProfilesManager.Instance.addProfile({ - type, - encoder, - profile, - builder - }) - - if (!self.transcodingProfiles[self.npmName]) self.transcodingProfiles[self.npmName] = [] - self.transcodingProfiles[self.npmName].push({ type, encoder, profile }) - - return true - } - - function addEncoderPriority (type: 'live' | 'vod', streamType: 'audio' | 'video', encoder: string, priority: number) { - VideoTranscodingProfilesManager.Instance.addEncoderPriority(type, streamType, encoder, priority) - - if (!self.transcodingEncoders[self.npmName]) self.transcodingEncoders[self.npmName] = [] - self.transcodingEncoders[self.npmName].push({ type, streamType, encoder, priority }) - } - - return { - addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { - return addProfile('live', encoder, profile, builder) - }, - - addVODProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { - return addProfile('vod', encoder, profile, builder) - }, - - addLiveEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { - return addEncoderPriority('live', streamType, encoder, priority) - }, - - addVODEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { - return addEncoderPriority('vod', streamType, encoder, priority) - }, - - removeAllProfilesAndEncoderPriorities () { - return self.reinitTranscodingProfilesAndEncoders(self.npmName) - } - } - } -} diff --git a/server/lib/plugins/theme-utils.ts b/server/lib/plugins/theme-utils.ts deleted file mode 100644 index 76c671f1c..000000000 --- a/server/lib/plugins/theme-utils.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { DEFAULT_THEME_NAME, DEFAULT_USER_THEME_NAME } from '../../initializers/constants' -import { PluginManager } from './plugin-manager' -import { CONFIG } from '../../initializers/config' - -function getThemeOrDefault (name: string, defaultTheme: string) { - if (isThemeRegistered(name)) return name - - // Fallback to admin default theme - if (name !== CONFIG.THEME.DEFAULT) return getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) - - return defaultTheme -} - -function isThemeRegistered (name: string) { - if (name === DEFAULT_THEME_NAME || name === DEFAULT_USER_THEME_NAME) return true - - return !!PluginManager.Instance.getRegisteredThemes() - .find(r => r.name === name) -} - -export { - getThemeOrDefault, - isThemeRegistered -} diff --git a/server/lib/plugins/video-constant-manager-factory.ts b/server/lib/plugins/video-constant-manager-factory.ts deleted file mode 100644 index 5f7edfbe2..000000000 --- a/server/lib/plugins/video-constant-manager-factory.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { - VIDEO_CATEGORIES, - VIDEO_LANGUAGES, - VIDEO_LICENCES, - VIDEO_PLAYLIST_PRIVACIES, - VIDEO_PRIVACIES -} from '@server/initializers/constants' -import { ConstantManager } from '@shared/models/plugins/server/plugin-constant-manager.model' - -type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy' -type VideoConstant = Record - -type UpdatedVideoConstant = { - [name in AlterableVideoConstant]: { - [ npmName: string]: { - added: VideoConstant[] - deleted: VideoConstant[] - } - } -} - -const constantsHash: { [key in AlterableVideoConstant]: VideoConstant } = { - language: VIDEO_LANGUAGES, - licence: VIDEO_LICENCES, - category: VIDEO_CATEGORIES, - privacy: VIDEO_PRIVACIES, - playlistPrivacy: VIDEO_PLAYLIST_PRIVACIES -} - -export class VideoConstantManagerFactory { - private readonly updatedVideoConstants: UpdatedVideoConstant = { - playlistPrivacy: { }, - privacy: { }, - language: { }, - licence: { }, - category: { } - } - - constructor ( - private readonly npmName: string - ) {} - - public resetVideoConstants (npmName: string) { - const types: AlterableVideoConstant[] = [ 'language', 'licence', 'category', 'privacy', 'playlistPrivacy' ] - for (const type of types) { - this.resetConstants({ npmName, type }) - } - } - - private resetConstants (parameters: { npmName: string, type: AlterableVideoConstant }) { - const { npmName, type } = parameters - const updatedConstants = this.updatedVideoConstants[type][npmName] - - if (!updatedConstants) return - - for (const added of updatedConstants.added) { - delete constantsHash[type][added.key] - } - - for (const deleted of updatedConstants.deleted) { - constantsHash[type][deleted.key] = deleted.label - } - - delete this.updatedVideoConstants[type][npmName] - } - - public createVideoConstantManager(type: AlterableVideoConstant): ConstantManager { - const { npmName } = this - return { - addConstant: (key: K, label: string) => this.addConstant({ npmName, type, key, label }), - deleteConstant: (key: K) => this.deleteConstant({ npmName, type, key }), - getConstantValue: (key: K) => constantsHash[type][key], - getConstants: () => constantsHash[type] as Record, - resetConstants: () => this.resetConstants({ npmName, type }) - } - } - - private addConstant (parameters: { - npmName: string - type: AlterableVideoConstant - key: T - label: string - }) { - const { npmName, type, key, label } = parameters - const obj = constantsHash[type] - - if (obj[key]) { - logger.warn('Cannot add %s %s by plugin %s: key already exists.', type, npmName, key) - return false - } - - if (!this.updatedVideoConstants[type][npmName]) { - this.updatedVideoConstants[type][npmName] = { - added: [], - deleted: [] - } - } - - this.updatedVideoConstants[type][npmName].added.push({ key, label } as VideoConstant) - obj[key] = label - - return true - } - - private deleteConstant (parameters: { - npmName: string - type: AlterableVideoConstant - key: T - }) { - const { npmName, type, key } = parameters - const obj = constantsHash[type] - - if (!obj[key]) { - logger.warn('Cannot delete %s by plugin %s: key %s does not exist.', type, npmName, key) - return false - } - - if (!this.updatedVideoConstants[type][npmName]) { - this.updatedVideoConstants[type][npmName] = { - added: [], - deleted: [] - } - } - - const updatedConstants = this.updatedVideoConstants[type][npmName] - - const alreadyAdded = updatedConstants.added.find(a => a.key === key) - if (alreadyAdded) { - updatedConstants.added.filter(a => a.key !== key) - } else if (obj[key]) { - updatedConstants.deleted.push({ key, label: obj[key] } as VideoConstant) - } - - delete obj[key] - - return true - } -} diff --git a/server/lib/plugins/yarn.ts b/server/lib/plugins/yarn.ts deleted file mode 100644 index 9cf6ec9e9..000000000 --- a/server/lib/plugins/yarn.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { outputJSON, pathExists } from 'fs-extra' -import { join } from 'path' -import { execShell } from '../../helpers/core-utils' -import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { getLatestPluginVersion } from './plugin-index' - -async function installNpmPlugin (npmName: string, versionArg?: string) { - // Security check - checkNpmPluginNameOrThrow(npmName) - if (versionArg) checkPluginVersionOrThrow(versionArg) - - const version = versionArg || await getLatestPluginVersion(npmName) - - let toInstall = npmName - if (version) toInstall += `@${version}` - - const { stdout } = await execYarn('add ' + toInstall) - - logger.debug('Added a yarn package.', { yarnStdout: stdout }) -} - -async function installNpmPluginFromDisk (path: string) { - await execYarn('add file:' + path) -} - -async function removeNpmPlugin (name: string) { - checkNpmPluginNameOrThrow(name) - - await execYarn('remove ' + name) -} - -async function rebuildNativePlugins () { - await execYarn('install --pure-lockfile') -} - -// ############################################################################ - -export { - installNpmPlugin, - installNpmPluginFromDisk, - rebuildNativePlugins, - removeNpmPlugin -} - -// ############################################################################ - -async function execYarn (command: string) { - try { - const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR - const pluginPackageJSON = join(pluginDirectory, 'package.json') - - // Create empty package.json file if needed - if (!await pathExists(pluginPackageJSON)) { - await outputJSON(pluginPackageJSON, {}) - } - - return execShell(`yarn ${command}`, { cwd: pluginDirectory }) - } catch (result) { - logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr }) - - throw result.err - } -} - -function checkNpmPluginNameOrThrow (name: string) { - if (!isNpmPluginNameValid(name)) throw new Error('Invalid NPM plugin name to install') -} - -function checkPluginVersionOrThrow (name: string) { - if (!isPluginStableOrUnstableVersionValid(name)) throw new Error('Invalid NPM plugin version to install') -} diff --git a/server/lib/redis.ts b/server/lib/redis.ts deleted file mode 100644 index 48d9986b5..000000000 --- a/server/lib/redis.ts +++ /dev/null @@ -1,465 +0,0 @@ -import IoRedis, { RedisOptions } from 'ioredis' -import { exists } from '@server/helpers/custom-validators/misc' -import { sha256 } from '@shared/extra-utils' -import { logger } from '../helpers/logger' -import { generateRandomString } from '../helpers/utils' -import { CONFIG } from '../initializers/config' -import { - AP_CLEANER, - CONTACT_FORM_LIFETIME, - RESUMABLE_UPLOAD_SESSION_LIFETIME, - TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, - EMAIL_VERIFY_LIFETIME, - USER_PASSWORD_CREATE_LIFETIME, - USER_PASSWORD_RESET_LIFETIME, - VIEW_LIFETIME, - WEBSERVER -} from '../initializers/constants' - -class Redis { - - private static instance: Redis - private initialized = false - private connected = false - private client: IoRedis - private prefix: string - - private constructor () { - } - - init () { - // Already initialized - if (this.initialized === true) return - this.initialized = true - - const redisMode = CONFIG.REDIS.SENTINEL.ENABLED ? 'sentinel' : 'standalone' - logger.info('Connecting to redis ' + redisMode + '...') - - this.client = new IoRedis(Redis.getRedisClientOptions('', { enableAutoPipelining: true })) - this.client.on('error', err => logger.error('Redis failed to connect', { err })) - this.client.on('connect', () => { - logger.info('Connected to redis.') - - this.connected = true - }) - this.client.on('reconnecting', (ms) => { - logger.error(`Reconnecting to redis in ${ms}.`) - }) - this.client.on('close', () => { - logger.error('Connection to redis has closed.') - this.connected = false - }) - - this.client.on('end', () => { - logger.error('Connection to redis has closed and no more reconnects will be done.') - }) - - this.prefix = 'redis-' + WEBSERVER.HOST + '-' - } - - static getRedisClientOptions (name?: string, options: RedisOptions = {}): RedisOptions { - const connectionName = [ 'PeerTube', name ].join('') - const connectTimeout = 20000 // Could be slow since node use sync call to compile PeerTube - - if (CONFIG.REDIS.SENTINEL.ENABLED) { - return { - connectionName, - connectTimeout, - enableTLSForSentinelMode: CONFIG.REDIS.SENTINEL.ENABLE_TLS, - sentinelPassword: CONFIG.REDIS.AUTH, - sentinels: CONFIG.REDIS.SENTINEL.SENTINELS, - name: CONFIG.REDIS.SENTINEL.MASTER_NAME, - ...options - } - } - - return { - connectionName, - connectTimeout, - password: CONFIG.REDIS.AUTH, - db: CONFIG.REDIS.DB, - host: CONFIG.REDIS.HOSTNAME, - port: CONFIG.REDIS.PORT, - path: CONFIG.REDIS.SOCKET, - showFriendlyErrorStack: true, - ...options - } - } - - getClient () { - return this.client - } - - getPrefix () { - return this.prefix - } - - isConnected () { - return this.connected - } - - /* ************ Forgot password ************ */ - - async setResetPasswordVerificationString (userId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_RESET_LIFETIME) - - return generatedString - } - - async setCreatePasswordVerificationString (userId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_CREATE_LIFETIME) - - return generatedString - } - - async removePasswordVerificationString (userId: number) { - return this.removeValue(this.generateResetPasswordKey(userId)) - } - - async getResetPasswordVerificationString (userId: number) { - return this.getValue(this.generateResetPasswordKey(userId)) - } - - /* ************ Two factor auth request ************ */ - - async setTwoFactorRequest (userId: number, otpSecret: string) { - const requestToken = await generateRandomString(32) - - await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME) - - return requestToken - } - - async getTwoFactorRequestToken (userId: number, requestToken: string) { - return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken)) - } - - /* ************ Email verification ************ */ - - async setUserVerifyEmailVerificationString (userId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateUserVerifyEmailKey(userId), generatedString, EMAIL_VERIFY_LIFETIME) - - return generatedString - } - - async getUserVerifyEmailLink (userId: number) { - return this.getValue(this.generateUserVerifyEmailKey(userId)) - } - - async setRegistrationVerifyEmailVerificationString (registrationId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateRegistrationVerifyEmailKey(registrationId), generatedString, EMAIL_VERIFY_LIFETIME) - - return generatedString - } - - async getRegistrationVerifyEmailLink (registrationId: number) { - return this.getValue(this.generateRegistrationVerifyEmailKey(registrationId)) - } - - /* ************ Contact form per IP ************ */ - - async setContactFormIp (ip: string) { - return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) - } - - async doesContactFormIpExist (ip: string) { - return this.exists(this.generateContactFormKey(ip)) - } - - /* ************ Views per IP ************ */ - - setIPVideoView (ip: string, videoUUID: string) { - return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) - } - - async doesVideoIPViewExist (ip: string, videoUUID: string) { - return this.exists(this.generateIPViewKey(ip, videoUUID)) - } - - /* ************ Video views stats ************ */ - - addVideoViewStats (videoId: number) { - const { videoKey, setKey } = this.generateVideoViewStatsKeys({ videoId }) - - return Promise.all([ - this.addToSet(setKey, videoId.toString()), - this.increment(videoKey) - ]) - } - - async getVideoViewsStats (videoId: number, hour: number) { - const { videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) - - const valueString = await this.getValue(videoKey) - const valueInt = parseInt(valueString, 10) - - if (isNaN(valueInt)) { - logger.error('Cannot get videos views stats of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString) - return undefined - } - - return valueInt - } - - async listVideosViewedForStats (hour: number) { - const { setKey } = this.generateVideoViewStatsKeys({ hour }) - - const stringIds = await this.getSet(setKey) - return stringIds.map(s => parseInt(s, 10)) - } - - deleteVideoViewsStats (videoId: number, hour: number) { - const { setKey, videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) - - return Promise.all([ - this.deleteFromSet(setKey, videoId.toString()), - this.deleteKey(videoKey) - ]) - } - - /* ************ Local video views buffer ************ */ - - addLocalVideoView (videoId: number) { - const { videoKey, setKey } = this.generateLocalVideoViewsKeys(videoId) - - return Promise.all([ - this.addToSet(setKey, videoId.toString()), - this.increment(videoKey) - ]) - } - - async getLocalVideoViews (videoId: number) { - const { videoKey } = this.generateLocalVideoViewsKeys(videoId) - - const valueString = await this.getValue(videoKey) - const valueInt = parseInt(valueString, 10) - - if (isNaN(valueInt)) { - logger.error('Cannot get videos views of video %d: views number is NaN (%s).', videoId, valueString) - return undefined - } - - return valueInt - } - - async listLocalVideosViewed () { - const { setKey } = this.generateLocalVideoViewsKeys() - - const stringIds = await this.getSet(setKey) - return stringIds.map(s => parseInt(s, 10)) - } - - deleteLocalVideoViews (videoId: number) { - const { setKey, videoKey } = this.generateLocalVideoViewsKeys(videoId) - - return Promise.all([ - this.deleteFromSet(setKey, videoId.toString()), - this.deleteKey(videoKey) - ]) - } - - /* ************ Video viewers stats ************ */ - - getLocalVideoViewer (options: { - key?: string - // Or - ip?: string - videoId?: number - }) { - if (options.key) return this.getObject(options.key) - - const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId) - - return this.getObject(viewerKey) - } - - setLocalVideoViewer (ip: string, videoId: number, object: any) { - const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId) - - return Promise.all([ - this.addToSet(setKey, viewerKey), - this.setObject(viewerKey, object) - ]) - } - - listLocalVideoViewerKeys () { - const { setKey } = this.generateLocalVideoViewerKeys() - - return this.getSet(setKey) - } - - deleteLocalVideoViewersKeys (key: string) { - const { setKey } = this.generateLocalVideoViewerKeys() - - return Promise.all([ - this.deleteFromSet(setKey, key), - this.deleteKey(key) - ]) - } - - /* ************ Resumable uploads final responses ************ */ - - setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) { - return this.setValue( - 'resumable-upload-' + uploadId, - response - ? JSON.stringify(response) - : '', - RESUMABLE_UPLOAD_SESSION_LIFETIME - ) - } - - doesUploadSessionExist (uploadId: string) { - return this.exists('resumable-upload-' + uploadId) - } - - async getUploadSession (uploadId: string) { - const value = await this.getValue('resumable-upload-' + uploadId) - - return value - ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } } - : undefined - } - - deleteUploadSession (uploadId: string) { - return this.deleteKey('resumable-upload-' + uploadId) - } - - /* ************ AP resource unavailability ************ */ - - async addAPUnavailability (url: string) { - const key = this.generateAPUnavailabilityKey(url) - - const value = await this.increment(key) - await this.setExpiration(key, AP_CLEANER.PERIOD * 2) - - return value - } - - /* ************ Keys generation ************ */ - - private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string } - private generateLocalVideoViewsKeys (): { setKey: string } - private generateLocalVideoViewsKeys (videoId?: number) { - return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } - } - - private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string } - private generateLocalVideoViewerKeys (): { setKey: string } - private generateLocalVideoViewerKeys (ip?: string, videoId?: number) { - return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` } - } - - private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { - const hour = exists(options.hour) - ? options.hour - : new Date().getHours() - - return { setKey: `videos-view-h${hour}`, videoKey: `video-view-${options.videoId}-h${hour}` } - } - - private generateResetPasswordKey (userId: number) { - return 'reset-password-' + userId - } - - private generateTwoFactorRequestKey (userId: number, token: string) { - return 'two-factor-request-' + userId + '-' + token - } - - private generateUserVerifyEmailKey (userId: number) { - return 'verify-email-user-' + userId - } - - private generateRegistrationVerifyEmailKey (registrationId: number) { - return 'verify-email-registration-' + registrationId - } - - private generateIPViewKey (ip: string, videoUUID: string) { - return `views-${videoUUID}-${ip}` - } - - private generateContactFormKey (ip: string) { - return 'contact-form-' + ip - } - - private generateAPUnavailabilityKey (url: string) { - return 'ap-unavailability-' + sha256(url) - } - - /* ************ Redis helpers ************ */ - - private getValue (key: string) { - return this.client.get(this.prefix + key) - } - - private getSet (key: string) { - return this.client.smembers(this.prefix + key) - } - - private addToSet (key: string, value: string) { - return this.client.sadd(this.prefix + key, value) - } - - private deleteFromSet (key: string, value: string) { - return this.client.srem(this.prefix + key, value) - } - - private deleteKey (key: string) { - return this.client.del(this.prefix + key) - } - - private async getObject (key: string) { - const value = await this.getValue(key) - if (!value) return null - - return JSON.parse(value) - } - - private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) { - return this.setValue(key, JSON.stringify(value), expirationMilliseconds) - } - - private async setValue (key: string, value: string, expirationMilliseconds?: number) { - const result = expirationMilliseconds !== undefined - ? await this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds) - : await this.client.set(this.prefix + key, value) - - if (result !== 'OK') throw new Error('Redis set result is not OK.') - } - - private removeValue (key: string) { - return this.client.del(this.prefix + key) - } - - private increment (key: string) { - return this.client.incr(this.prefix + key) - } - - private async exists (key: string) { - const result = await this.client.exists(this.prefix + key) - - return result !== 0 - } - - private setExpiration (key: string, ms: number) { - return this.client.expire(this.prefix + key, ms / 1000) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - Redis -} diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts deleted file mode 100644 index 2613d01be..000000000 --- a/server/lib/redundancy.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Transaction } from 'sequelize' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { getServerActor } from '@server/models/application/application' -import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models' -import { Activity } from '@shared/models' -import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' -import { sendUndoCacheFile } from './activitypub/send' - -const lTags = loggerTagsFactory('redundancy') - -async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { - const serverActor = await getServerActor() - - // Local cache, send undo to remote instances - if (videoRedundancy.actorId === serverActor.id) await sendUndoCacheFile(serverActor, videoRedundancy, t) - - await videoRedundancy.destroy({ transaction: t }) -} - -async function removeRedundanciesOfServer (serverId: number) { - const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) - - for (const redundancy of redundancies) { - await removeVideoRedundancy(redundancy) - } -} - -async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) { - const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM - if (configAcceptFrom === 'nobody') { - logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id, lTags()) - return false - } - - if (configAcceptFrom === 'followings') { - const serverActor = await getServerActor() - const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id) - - if (allowed !== true) { - logger.info( - 'Do not accept remote redundancy %s because actor %s is not followed by our instance.', - activity.id, byActor.url, lTags() - ) - return false - } - } - - return true -} - -// --------------------------------------------------------------------------- - -export { - isRedundancyAccepted, - removeRedundanciesOfServer, - removeVideoRedundancy -} diff --git a/server/lib/runners/index.ts b/server/lib/runners/index.ts deleted file mode 100644 index a737c7b59..000000000 --- a/server/lib/runners/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './job-handlers' -export * from './runner' -export * from './runner-urls' diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts deleted file mode 100644 index 329977de1..000000000 --- a/server/lib/runners/job-handlers/abstract-job-handler.ts +++ /dev/null @@ -1,269 +0,0 @@ -import { throttle } from 'lodash' -import { saveInTransactionWithRetries } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { RUNNER_JOBS } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { PeerTubeSocket } from '@server/lib/peertube-socket' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { setAsUpdated } from '@server/models/shared' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { - RunnerJobLiveRTMPHLSTranscodingPayload, - RunnerJobLiveRTMPHLSTranscodingPrivatePayload, - RunnerJobState, - RunnerJobStudioTranscodingPayload, - RunnerJobSuccessPayload, - RunnerJobType, - RunnerJobUpdatePayload, - RunnerJobVideoStudioTranscodingPrivatePayload, - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODAudioMergeTranscodingPrivatePayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODHLSTranscodingPrivatePayload, - RunnerJobVODWebVideoTranscodingPayload, - RunnerJobVODWebVideoTranscodingPrivatePayload -} from '@shared/models' - -type CreateRunnerJobArg = - { - type: Extract - payload: RunnerJobVODWebVideoTranscodingPayload - privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobVODHLSTranscodingPayload - privatePayload: RunnerJobVODHLSTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobVODAudioMergeTranscodingPayload - privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobLiveRTMPHLSTranscodingPayload - privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobStudioTranscodingPayload - privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload - } - -export abstract class AbstractJobHandler { - - protected readonly lTags = loggerTagsFactory('runner') - - static setJobAsUpdatedThrottled = throttle(setAsUpdated, 2000) - - // --------------------------------------------------------------------------- - - abstract create (options: C): Promise - - protected async createRunnerJob (options: CreateRunnerJobArg & { - jobUUID: string - priority: number - dependsOnRunnerJob?: MRunnerJob - }): Promise { - const { priority, dependsOnRunnerJob } = options - - logger.debug('Creating runner job', { options, ...this.lTags(options.type) }) - - const runnerJob = new RunnerJobModel({ - ...pick(options, [ 'type', 'payload', 'privatePayload' ]), - - uuid: options.jobUUID, - - state: dependsOnRunnerJob - ? RunnerJobState.WAITING_FOR_PARENT_JOB - : RunnerJobState.PENDING, - - dependsOnRunnerJobId: dependsOnRunnerJob?.id, - - priority - }) - - const job = await sequelizeTypescript.transaction(async transaction => { - return runnerJob.save({ transaction }) - }) - - if (runnerJob.state === RunnerJobState.PENDING) { - PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() - } - - return job - } - - // --------------------------------------------------------------------------- - - protected abstract specificUpdate (options: { - runnerJob: MRunnerJob - updatePayload?: U - }): Promise | void - - async update (options: { - runnerJob: MRunnerJob - progress?: number - updatePayload?: U - }) { - const { runnerJob, progress } = options - - await this.specificUpdate(options) - - if (progress) runnerJob.progress = progress - - if (!runnerJob.changed()) { - try { - await AbstractJobHandler.setJobAsUpdatedThrottled({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id }) - } catch (err) { - logger.warn('Cannot set remote job as updated', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) - } - - return - } - - await saveInTransactionWithRetries(runnerJob) - } - - // --------------------------------------------------------------------------- - - async complete (options: { - runnerJob: MRunnerJob - resultPayload: S - }) { - const { runnerJob } = options - - runnerJob.state = RunnerJobState.COMPLETING - await saveInTransactionWithRetries(runnerJob) - - try { - await this.specificComplete(options) - - runnerJob.state = RunnerJobState.COMPLETED - } catch (err) { - logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) - - runnerJob.state = RunnerJobState.ERRORED - runnerJob.error = err.message - } - - runnerJob.progress = null - runnerJob.finishedAt = new Date() - - await saveInTransactionWithRetries(runnerJob) - - const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob) - - if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() - } - - protected abstract specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: S - }): Promise | void - - // --------------------------------------------------------------------------- - - async cancel (options: { - runnerJob: MRunnerJob - fromParent?: boolean - }) { - const { runnerJob, fromParent } = options - - await this.specificCancel(options) - - const cancelState = fromParent - ? RunnerJobState.PARENT_CANCELLED - : RunnerJobState.CANCELLED - - runnerJob.setToErrorOrCancel(cancelState) - - await saveInTransactionWithRetries(runnerJob) - - const children = await RunnerJobModel.listChildrenOf(runnerJob) - for (const child of children) { - logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid)) - - await this.cancel({ runnerJob: child, fromParent: true }) - } - } - - protected abstract specificCancel (options: { - runnerJob: MRunnerJob - }): Promise | void - - // --------------------------------------------------------------------------- - - protected abstract isAbortSupported (): boolean - - async abort (options: { - runnerJob: MRunnerJob - }) { - const { runnerJob } = options - - if (this.isAbortSupported() !== true) { - return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' }) - } - - await this.specificAbort(options) - - runnerJob.resetToPending() - - await saveInTransactionWithRetries(runnerJob) - } - - protected setAbortState (runnerJob: MRunnerJob) { - runnerJob.resetToPending() - } - - protected abstract specificAbort (options: { - runnerJob: MRunnerJob - }): Promise | void - - // --------------------------------------------------------------------------- - - async error (options: { - runnerJob: MRunnerJob - message: string - fromParent?: boolean - }) { - const { runnerJob, message, fromParent } = options - - const errorState = fromParent - ? RunnerJobState.PARENT_ERRORED - : RunnerJobState.ERRORED - - const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES - ? RunnerJobState.PENDING - : errorState - - await this.specificError({ ...options, nextState }) - - if (nextState === errorState) { - runnerJob.setToErrorOrCancel(nextState) - runnerJob.error = message - } else { - runnerJob.resetToPending() - } - - await saveInTransactionWithRetries(runnerJob) - - if (runnerJob.state === errorState) { - const children = await RunnerJobModel.listChildrenOf(runnerJob) - - for (const child of children) { - logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid)) - - await this.error({ runnerJob: child, message: 'Parent error', fromParent: true }) - } - } - } - - protected abstract specificError (options: { - runnerJob: MRunnerJob - message: string - nextState: RunnerJobState - }): Promise | void -} diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts deleted file mode 100644 index f425828d9..000000000 --- a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts +++ /dev/null @@ -1,66 +0,0 @@ - -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MRunnerJob } from '@server/types/models/runners' -import { RunnerJobState, RunnerJobSuccessPayload, RunnerJobUpdatePayload, RunnerJobVODPrivatePayload } from '@shared/models' -import { AbstractJobHandler } from './abstract-job-handler' -import { loadTranscodingRunnerVideo } from './shared' - -// eslint-disable-next-line max-len -export abstract class AbstractVODTranscodingJobHandler extends AbstractJobHandler { - - protected isAbortSupported () { - return true - } - - protected specificUpdate (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected specificAbort (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected async specificError (options: { - runnerJob: MRunnerJob - nextState: RunnerJobState - }) { - if (options.nextState !== RunnerJobState.ERRORED) return - - const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) - if (!video) return - - await moveToFailedTranscodingState(video) - - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - } - - protected async specificCancel (options: { - runnerJob: MRunnerJob - }) { - const { runnerJob } = options - - const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) - if (!video) return - - const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid)) - - if (pending === 0) { - logger.info( - `All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`, - this.lTags(video.uuid) - ) - - const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload - await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo }) - } - } -} diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts deleted file mode 100644 index 40ad2f97a..000000000 --- a/server/lib/runners/job-handlers/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './abstract-job-handler' -export * from './live-rtmp-hls-transcoding-job-handler' -export * from './runner-job-handlers' -export * from './video-studio-transcoding-job-handler' -export * from './vod-audio-merge-transcoding-job-handler' -export * from './vod-hls-transcoding-job-handler' -export * from './vod-web-video-transcoding-job-handler' diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts deleted file mode 100644 index 6b2894f8c..000000000 --- a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { move, remove } from 'fs-extra' -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { JOB_PRIORITY } from '@server/initializers/constants' -import { LiveManager } from '@server/lib/live' -import { MStreamingPlaylist, MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { buildUUID } from '@shared/extra-utils' -import { - LiveRTMPHLSTranscodingSuccess, - LiveRTMPHLSTranscodingUpdatePayload, - LiveVideoError, - RunnerJobLiveRTMPHLSTranscodingPayload, - RunnerJobLiveRTMPHLSTranscodingPrivatePayload, - RunnerJobState -} from '@shared/models' -import { AbstractJobHandler } from './abstract-job-handler' - -type CreateOptions = { - video: MVideo - playlist: MStreamingPlaylist - - sessionId: string - rtmpUrl: string - - toTranscode: { - resolution: number - fps: number - }[] - - segmentListSize: number - segmentDuration: number - - outputDirectory: string -} - -// eslint-disable-next-line max-len -export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler { - - async create (options: CreateOptions) { - const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory, sessionId } = options - - const jobUUID = buildUUID() - const payload: RunnerJobLiveRTMPHLSTranscodingPayload = { - input: { - rtmpUrl - }, - output: { - toTranscode, - segmentListSize, - segmentDuration - } - } - - const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = { - videoUUID: video.uuid, - masterPlaylistName: playlist.playlistFilename, - sessionId, - outputDirectory - } - - const job = await this.createRunnerJob({ - type: 'live-rtmp-hls-transcoding', - jobUUID, - payload, - privatePayload, - priority: JOB_PRIORITY.TRANSCODING - }) - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificUpdate (options: { - runnerJob: MRunnerJob - updatePayload: LiveRTMPHLSTranscodingUpdatePayload - }) { - const { runnerJob, updatePayload } = options - - const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload - const outputDirectory = privatePayload.outputDirectory - const videoUUID = privatePayload.videoUUID - - // Always process the chunk first before moving m3u8 that references this chunk - if (updatePayload.type === 'add-chunk') { - await move( - updatePayload.videoChunkFile as string, - join(outputDirectory, updatePayload.videoChunkFilename), - { overwrite: true } - ) - } else if (updatePayload.type === 'remove-chunk') { - await remove(join(outputDirectory, updatePayload.videoChunkFilename)) - } - - if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) { - await move( - updatePayload.resolutionPlaylistFile as string, - join(outputDirectory, updatePayload.resolutionPlaylistFilename), - { overwrite: true } - ) - } - - if (updatePayload.masterPlaylistFile) { - await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true }) - } - - logger.debug( - 'Runner live RTMP to HLS job %s for %s updated.', - runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) } - ) - } - - // --------------------------------------------------------------------------- - - protected specificComplete (options: { - runnerJob: MRunnerJob - }) { - return this.stopLive({ - runnerJob: options.runnerJob, - type: 'ended' - }) - } - - // --------------------------------------------------------------------------- - - protected isAbortSupported () { - return false - } - - protected specificAbort () { - throw new Error('Not implemented') - } - - protected specificError (options: { - runnerJob: MRunnerJob - nextState: RunnerJobState - }) { - return this.stopLive({ - runnerJob: options.runnerJob, - type: 'errored' - }) - } - - protected specificCancel (options: { - runnerJob: MRunnerJob - }) { - return this.stopLive({ - runnerJob: options.runnerJob, - type: 'cancelled' - }) - } - - private stopLive (options: { - runnerJob: MRunnerJob - type: 'ended' | 'errored' | 'cancelled' - }) { - const { runnerJob, type } = options - - const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload - const videoUUID = privatePayload.videoUUID - - const errorType = { - ended: null, - errored: LiveVideoError.RUNNER_JOB_ERROR, - cancelled: LiveVideoError.RUNNER_JOB_CANCEL - } - - LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type]) - - logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID)) - } -} diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts deleted file mode 100644 index 85551c365..000000000 --- a/server/lib/runners/job-handlers/runner-job-handlers.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { MRunnerJob } from '@server/types/models/runners' -import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' -import { AbstractJobHandler } from './abstract-job-handler' -import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' -import { VideoStudioTranscodingJobHandler } from './video-studio-transcoding-job-handler' -import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' -import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' -import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' - -const processors: Record AbstractJobHandler> = { - 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, - 'vod-hls-transcoding': VODHLSTranscodingJobHandler, - 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, - 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler, - 'video-studio-transcoding': VideoStudioTranscodingJobHandler -} - -export function getRunnerJobHandlerClass (job: MRunnerJob) { - return processors[job.type] -} diff --git a/server/lib/runners/job-handlers/shared/index.ts b/server/lib/runners/job-handlers/shared/index.ts deleted file mode 100644 index 348273ae2..000000000 --- a/server/lib/runners/job-handlers/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './vod-helpers' diff --git a/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/lib/runners/job-handlers/shared/vod-helpers.ts deleted file mode 100644 index 1a2ad02ca..000000000 --- a/server/lib/runners/job-handlers/shared/vod-helpers.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { move } from 'fs-extra' -import { dirname, join } from 'path' -import { logger, LoggerTagsFn } from '@server/helpers/logger' -import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' -import { onWebVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding' -import { buildNewFile } from '@server/lib/video-file' -import { VideoModel } from '@server/models/video/video' -import { MVideoFullLight } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@shared/models' - -export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { - video: MVideoFullLight - videoFilePath: string - privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload -}) { - const { video, videoFilePath, privatePayload } = options - - const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' }) - videoFile.videoId = video.id - - const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) - await move(videoFilePath, newVideoFilePath) - - await onWebVideoFileTranscoding({ - video, - videoFile, - videoOutputPath: newVideoFilePath - }) - - await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) -} - -export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) { - const videoUUID = runnerJob.privatePayload.videoUUID - - const video = await VideoModel.loadFull(videoUUID) - if (!video) { - logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID)) - return undefined - } - - return video -} diff --git a/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts b/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts deleted file mode 100644 index f604382b7..000000000 --- a/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts +++ /dev/null @@ -1,157 +0,0 @@ - -import { basename } from 'path' -import { logger } from '@server/helpers/logger' -import { onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { buildUUID } from '@shared/extra-utils' -import { - isVideoStudioTaskIntro, - isVideoStudioTaskOutro, - isVideoStudioTaskWatermark, - RunnerJobState, - RunnerJobUpdatePayload, - RunnerJobStudioTranscodingPayload, - RunnerJobVideoStudioTranscodingPrivatePayload, - VideoStudioTranscodingSuccess, - VideoState, - VideoStudioTaskPayload -} from '@shared/models' -import { generateRunnerEditionTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' -import { AbstractJobHandler } from './abstract-job-handler' -import { loadTranscodingRunnerVideo } from './shared' - -type CreateOptions = { - video: MVideo - tasks: VideoStudioTaskPayload[] - priority: number -} - -// eslint-disable-next-line max-len -export class VideoStudioTranscodingJobHandler extends AbstractJobHandler { - - async create (options: CreateOptions) { - const { video, priority, tasks } = options - - const jobUUID = buildUUID() - const payload: RunnerJobStudioTranscodingPayload = { - input: { - videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) - }, - tasks: tasks.map(t => { - if (isVideoStudioTaskIntro(t) || isVideoStudioTaskOutro(t)) { - return { - ...t, - - options: { - ...t.options, - - file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) - } - } - } - - if (isVideoStudioTaskWatermark(t)) { - return { - ...t, - - options: { - ...t.options, - - file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) - } - } - } - - return t - }) - } - - const privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload = { - videoUUID: video.uuid, - originalTasks: tasks - } - - const job = await this.createRunnerJob({ - type: 'video-studio-transcoding', - jobUUID, - payload, - privatePayload, - priority - }) - - return job - } - - // --------------------------------------------------------------------------- - - protected isAbortSupported () { - return true - } - - protected specificUpdate (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected specificAbort (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VideoStudioTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) { - await safeCleanupStudioTMPFiles(privatePayload.originalTasks) - - } - - const videoFilePath = resultPayload.videoFile as string - - await onVideoStudioEnded({ video, editionResultPath: videoFilePath, tasks: privatePayload.originalTasks }) - - logger.info( - 'Runner video edition transcoding job %s for %s ended.', - runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) - ) - } - - protected specificError (options: { - runnerJob: MRunnerJob - nextState: RunnerJobState - }) { - if (options.nextState === RunnerJobState.ERRORED) { - return this.specificErrorOrCancel(options) - } - - return Promise.resolve() - } - - protected specificCancel (options: { - runnerJob: MRunnerJob - }) { - return this.specificErrorOrCancel(options) - } - - private async specificErrorOrCancel (options: { - runnerJob: MRunnerJob - }) { - const { runnerJob } = options - - const payload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload - await safeCleanupStudioTMPFiles(payload.originalTasks) - - const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) - if (!video) return - - return video.setNewState(VideoState.PUBLISHED, false, undefined) - } -} diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts deleted file mode 100644 index 137a94535..000000000 --- a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { getVideoStreamDuration } from '@shared/ffmpeg' -import { - RunnerJobUpdatePayload, - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODWebVideoTranscodingPrivatePayload, - VODAudioMergeTranscodingSuccess -} from '@shared/models' -import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls' -import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' -import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' - -type CreateOptions = { - video: MVideo - isNewVideo: boolean - resolution: number - fps: number - priority: number - dependsOnRunnerJob?: MRunnerJob -} - -// eslint-disable-next-line max-len -export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler { - - async create (options: CreateOptions) { - const { video, resolution, fps, priority, dependsOnRunnerJob } = options - - const jobUUID = buildUUID() - const payload: RunnerJobVODAudioMergeTranscodingPayload = { - input: { - audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid), - previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid) - }, - output: { - resolution, - fps - } - } - - const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo' ]), - - videoUUID: video.uuid - } - - const job = await this.createRunnerJob({ - type: 'vod-audio-merge-transcoding', - jobUUID, - payload, - privatePayload, - priority, - dependsOnRunnerJob - }) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VODAudioMergeTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) return - - const videoFilePath = resultPayload.videoFile as string - - // ffmpeg generated a new video file, so update the video duration - // See https://trac.ffmpeg.org/ticket/5456 - video.duration = await getVideoStreamDuration(videoFilePath) - await video.save() - - // We can remove the old audio file - const oldAudioFile = video.VideoFiles[0] - await video.removeWebVideoFile(oldAudioFile) - await oldAudioFile.destroy() - video.VideoFiles = [] - - await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) - - logger.info( - 'Runner VOD audio merge transcoding job %s for %s ended.', - runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) - ) - } -} diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts deleted file mode 100644 index 02845952c..000000000 --- a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { move } from 'fs-extra' -import { dirname, join } from 'path' -import { logger } from '@server/helpers/logger' -import { renameVideoFileInPlaylist } from '@server/lib/hls' -import { getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' -import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding' -import { buildNewFile, removeAllWebVideoFiles } from '@server/lib/video-file' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { - RunnerJobUpdatePayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODHLSTranscodingPrivatePayload, - VODHLSTranscodingSuccess -} from '@shared/models' -import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' -import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' -import { loadTranscodingRunnerVideo } from './shared' - -type CreateOptions = { - video: MVideo - isNewVideo: boolean - deleteWebVideoFiles: boolean - resolution: number - fps: number - priority: number - dependsOnRunnerJob?: MRunnerJob -} - -// eslint-disable-next-line max-len -export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler { - - async create (options: CreateOptions) { - const { video, resolution, fps, dependsOnRunnerJob, priority } = options - - const jobUUID = buildUUID() - - const payload: RunnerJobVODHLSTranscodingPayload = { - input: { - videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) - }, - output: { - resolution, - fps - } - } - - const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]), - - videoUUID: video.uuid - } - - const job = await this.createRunnerJob({ - type: 'vod-hls-transcoding', - jobUUID, - payload, - privatePayload, - priority, - dependsOnRunnerJob - }) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VODHLSTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) return - - const videoFilePath = resultPayload.videoFile as string - const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string - - const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' }) - const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) - await move(videoFilePath, newVideoFilePath) - - const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename) - const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename) - await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath) - - await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename) - - await onHLSVideoFileTranscoding({ - video, - videoFile, - m3u8OutputPath: newResolutionPlaylistFilePath, - videoOutputPath: newVideoFilePath - }) - - await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) - - if (privatePayload.deleteWebVideoFiles === true) { - logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) - - await removeAllWebVideoFiles(video) - } - - logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) - } -} diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts deleted file mode 100644 index 9ee8ab88e..000000000 --- a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { - RunnerJobUpdatePayload, - RunnerJobVODWebVideoTranscodingPayload, - RunnerJobVODWebVideoTranscodingPrivatePayload, - VODWebVideoTranscodingSuccess -} from '@shared/models' -import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' -import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' -import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' - -type CreateOptions = { - video: MVideo - isNewVideo: boolean - resolution: number - fps: number - priority: number - dependsOnRunnerJob?: MRunnerJob -} - -// eslint-disable-next-line max-len -export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler { - - async create (options: CreateOptions) { - const { video, resolution, fps, priority, dependsOnRunnerJob } = options - - const jobUUID = buildUUID() - const payload: RunnerJobVODWebVideoTranscodingPayload = { - input: { - videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) - }, - output: { - resolution, - fps - } - } - - const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo' ]), - - videoUUID: video.uuid - } - - const job = await this.createRunnerJob({ - type: 'vod-web-video-transcoding', - jobUUID, - payload, - privatePayload, - dependsOnRunnerJob, - priority - }) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VODWebVideoTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) return - - const videoFilePath = resultPayload.videoFile as string - - await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) - - logger.info( - 'Runner VOD web video transcoding job %s for %s ended.', - runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) - ) - } -} diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts deleted file mode 100644 index a27060b33..000000000 --- a/server/lib/runners/runner-urls.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { WEBSERVER } from '@server/initializers/constants' - -export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) { - return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality' -} - -export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { - return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' -} - -export function generateRunnerEditionTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string, filename: string) { - return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/studio/task-files/' + filename -} diff --git a/server/lib/runners/runner.ts b/server/lib/runners/runner.ts deleted file mode 100644 index 947fdb3f0..000000000 --- a/server/lib/runners/runner.ts +++ /dev/null @@ -1,49 +0,0 @@ -import express from 'express' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' -import { MRunner, MRunnerJob } from '@server/types/models/runners' -import { RUNNER_JOBS } from '@server/initializers/constants' -import { RunnerJobState } from '@shared/models' - -const lTags = loggerTagsFactory('runner') - -const updatingRunner = new Set() - -function updateLastRunnerContact (req: express.Request, runner: MRunner) { - const now = new Date() - - // Don't update last runner contact too often - if (now.getTime() - runner.lastContact.getTime() < RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL) return - if (updatingRunner.has(runner.id)) return - - updatingRunner.add(runner.id) - - runner.lastContact = now - runner.ip = req.ip - - logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name)) - - retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - return runner.save({ transaction }) - }) - }) - .catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) })) - .finally(() => updatingRunner.delete(runner.id)) -} - -function runnerJobCanBeCancelled (runnerJob: MRunnerJob) { - const allowedStates = new Set([ - RunnerJobState.PENDING, - RunnerJobState.PROCESSING, - RunnerJobState.WAITING_FOR_PARENT_JOB - ]) - - return allowedStates.has(runnerJob.state) -} - -export { - updateLastRunnerContact, - runnerJobCanBeCancelled -} diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts deleted file mode 100644 index f3d51a22e..000000000 --- a/server/lib/schedulers/abstract-scheduler.ts +++ /dev/null @@ -1,35 +0,0 @@ -import Bluebird from 'bluebird' -import { logger } from '../../helpers/logger' - -export abstract class AbstractScheduler { - - protected abstract schedulerIntervalMs: number - - private interval: NodeJS.Timer - private isRunning = false - - enable () { - if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') - - this.interval = setInterval(() => this.execute(), this.schedulerIntervalMs) - } - - disable () { - clearInterval(this.interval) - } - - async execute () { - if (this.isRunning === true) return - this.isRunning = true - - try { - await this.internalExecute() - } catch (err) { - logger.error('Cannot execute %s scheduler.', this.constructor.name, { err }) - } finally { - this.isRunning = false - } - } - - protected abstract internalExecute (): Promise | Bluebird -} diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts deleted file mode 100644 index e1c56c135..000000000 --- a/server/lib/schedulers/actor-follow-scheduler.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { isTestOrDevInstance } from '../../helpers/core-utils' -import { logger } from '../../helpers/logger' -import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { ActorFollowModel } from '../../models/actor/actor-follow' -import { ActorFollowHealthCache } from '../actor-follow-health-cache' -import { AbstractScheduler } from './abstract-scheduler' - -export class ActorFollowScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES - - private constructor () { - super() - } - - protected async internalExecute () { - await this.processPendingScores() - - await this.removeBadActorFollows() - } - - private async processPendingScores () { - const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore() - const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds() - const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds() - - ActorFollowHealthCache.Instance.clearPendingFollowsScore() - ActorFollowHealthCache.Instance.clearBadFollowingServerIds() - ActorFollowHealthCache.Instance.clearGoodFollowingServerIds() - - for (const inbox of Object.keys(pendingScores)) { - await ActorFollowModel.updateScore(inbox, pendingScores[inbox]) - } - - await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY) - await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS) - } - - private async removeBadActorFollows () { - if (!isTestOrDevInstance()) logger.info('Removing bad actor follows (scheduler).') - - try { - await ActorFollowModel.removeBadActorFollows() - } catch (err) { - logger.error('Error in bad actor follows scheduler.', { err }) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts deleted file mode 100644 index 956ece749..000000000 --- a/server/lib/schedulers/auto-follow-index-instances.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { chunk } from 'lodash' -import { doJSONRequest } from '@server/helpers/requests' -import { JobQueue } from '@server/lib/job-queue' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { getServerActor } from '@server/models/application/application' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class AutoFollowIndexInstances extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES - - private lastCheck: Date - - private constructor () { - super() - } - - protected async internalExecute () { - return this.autoFollow() - } - - private async autoFollow () { - if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return - - const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL - - logger.info('Auto follow instances of index %s.', indexUrl) - - try { - const serverActor = await getServerActor() - - const searchParams = { count: 1000 } - if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() }) - - this.lastCheck = new Date() - - const { body } = await doJSONRequest(indexUrl, { searchParams }) - if (!body.data || Array.isArray(body.data) === false) { - logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body }) - return - } - - const hosts: string[] = body.data.map(o => o.host) - const chunks = chunk(hosts, 20) - - for (const chunk of chunks) { - const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk) - - for (const unfollowedHost of unfollowedHosts) { - const payload = { - host: unfollowedHost, - name: SERVER_ACTOR_NAME, - followerActorId: serverActor.id, - isAutoFollow: true - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) - } - } - - } catch (err) { - logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err }) - } - - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts deleted file mode 100644 index b06f5a9b5..000000000 --- a/server/lib/schedulers/geo-ip-update-scheduler.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { GeoIP } from '@server/helpers/geo-ip' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class GeoIPUpdateScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE - - private constructor () { - super() - } - - protected internalExecute () { - return GeoIP.Instance.updateDatabase() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/peertube-version-check-scheduler.ts b/server/lib/schedulers/peertube-version-check-scheduler.ts deleted file mode 100644 index bc38ed49f..000000000 --- a/server/lib/schedulers/peertube-version-check-scheduler.ts +++ /dev/null @@ -1,55 +0,0 @@ - -import { doJSONRequest } from '@server/helpers/requests' -import { ApplicationModel } from '@server/models/application/application' -import { compareSemVer } from '@shared/core-utils' -import { JoinPeerTubeVersions } from '@shared/models' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { PEERTUBE_VERSION, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { Notifier } from '../notifier' -import { AbstractScheduler } from './abstract-scheduler' - -export class PeerTubeVersionCheckScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION - - private constructor () { - super() - } - - protected async internalExecute () { - return this.checkLatestVersion() - } - - private async checkLatestVersion () { - if (CONFIG.PEERTUBE.CHECK_LATEST_VERSION.ENABLED === false) return - - logger.info('Checking latest PeerTube version.') - - const { body } = await doJSONRequest(CONFIG.PEERTUBE.CHECK_LATEST_VERSION.URL) - - if (!body?.peertube?.latestVersion) { - logger.warn('Cannot check latest PeerTube version: body is invalid.', { body }) - return - } - - const latestVersion = body.peertube.latestVersion - const application = await ApplicationModel.load() - - // Already checked this version - if (application.latestPeerTubeVersion === latestVersion) return - - if (compareSemVer(PEERTUBE_VERSION, latestVersion) < 0) { - application.latestPeerTubeVersion = latestVersion - await application.save() - - Notifier.Instance.notifyOfNewPeerTubeVersion(application, latestVersion) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/plugins-check-scheduler.ts b/server/lib/schedulers/plugins-check-scheduler.ts deleted file mode 100644 index 820c01693..000000000 --- a/server/lib/schedulers/plugins-check-scheduler.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { chunk } from 'lodash' -import { compareSemVer } from '@shared/core-utils' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { PluginModel } from '../../models/server/plugin' -import { Notifier } from '../notifier' -import { getLatestPluginsVersion } from '../plugins/plugin-index' -import { AbstractScheduler } from './abstract-scheduler' - -export class PluginsCheckScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PLUGINS - - private constructor () { - super() - } - - protected async internalExecute () { - return this.checkLatestPluginsVersion() - } - - private async checkLatestPluginsVersion () { - if (CONFIG.PLUGINS.INDEX.ENABLED === false) return - - logger.info('Checking latest plugins version.') - - const plugins = await PluginModel.listInstalled() - - // Process 10 plugins in 1 HTTP request - const chunks = chunk(plugins, 10) - for (const chunk of chunks) { - // Find plugins according to their npm name - const pluginIndex: { [npmName: string]: PluginModel } = {} - for (const plugin of chunk) { - pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin - } - - const npmNames = Object.keys(pluginIndex) - - try { - const results = await getLatestPluginsVersion(npmNames) - - for (const result of results) { - const plugin = pluginIndex[result.npmName] - if (!result.latestVersion) continue - - if ( - !plugin.latestVersion || - (plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0) - ) { - plugin.latestVersion = result.latestVersion - await plugin.save() - - // Notify if there is an higher plugin version available - if (compareSemVer(plugin.version, result.latestVersion) < 0) { - Notifier.Instance.notifyOfNewPluginVersion(plugin) - } - - logger.info('Plugin %s has a new latest version %s.', result.npmName, plugin.latestVersion) - } - } - } catch (err) { - logger.error('Cannot get latest plugins version.', { npmNames, err }) - } - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts deleted file mode 100644 index 61e93eafa..000000000 --- a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts +++ /dev/null @@ -1,40 +0,0 @@ - -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' -import { uploadx } from '../uploadx' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') - -export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - private lastExecutionTimeMs: number - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS - - private constructor () { - super() - - this.lastExecutionTimeMs = new Date().getTime() - } - - protected async internalExecute () { - logger.debug('Removing dangling resumable uploads', lTags()) - - const now = new Date().getTime() - - try { - // Remove files that were not updated since the last execution - await uploadx.storage.purge(now - this.lastExecutionTimeMs) - } catch (error) { - logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) - } finally { - this.lastExecutionTimeMs = now - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts deleted file mode 100644 index 34b160799..000000000 --- a/server/lib/schedulers/remove-old-history-scheduler.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { logger } from '../../helpers/logger' -import { AbstractScheduler } from './abstract-scheduler' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { UserVideoHistoryModel } from '../../models/user/user-video-history' -import { CONFIG } from '../../initializers/config' - -export class RemoveOldHistoryScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY - - private constructor () { - super() - } - - protected internalExecute () { - if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return - - logger.info('Removing old videos history.') - - const now = new Date() - const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString() - - return UserVideoHistoryModel.removeOldHistory(beforeDate) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts deleted file mode 100644 index 8bc53a045..000000000 --- a/server/lib/schedulers/remove-old-views-scheduler.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { VideoViewModel } from '@server/models/view/video-view' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class RemoveOldViewsScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS - - private constructor () { - super() - } - - protected internalExecute () { - if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return - - logger.info('Removing old videos views.') - - const now = new Date() - const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString() - - return VideoViewModel.removeOldRemoteViewsHistory(beforeDate) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts deleted file mode 100644 index f7a26d2bc..000000000 --- a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { logger, loggerTagsFactory } from '../../helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { getRunnerJobHandlerClass } from '../runners' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('runner') - -export class RunnerJobWatchDogScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG - - private constructor () { - super() - } - - protected async internalExecute () { - const vodStalledJobs = await RunnerJobModel.listStalledJobs({ - staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, - types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ] - }) - - const liveStalledJobs = await RunnerJobModel.listStalledJobs({ - staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE, - types: [ 'live-rtmp-hls-transcoding' ] - }) - - for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) { - logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type)) - - const Handler = getRunnerJobHandlerClass(stalled) - await new Handler().abort({ runnerJob: stalled }) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts deleted file mode 100644 index e38685c04..000000000 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { MScheduleVideoUpdate } from '@server/types/models' -import { VideoPrivacy, VideoState } from '@shared/models' -import { logger } from '../../helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { sequelizeTypescript } from '../../initializers/database' -import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' -import { Notifier } from '../notifier' -import { addVideoJobsAfterUpdate } from '../video' -import { VideoPathManager } from '../video-path-manager' -import { setVideoPrivacy } from '../video-privacy' -import { AbstractScheduler } from './abstract-scheduler' - -export class UpdateVideosScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS - - private constructor () { - super() - } - - protected async internalExecute () { - return this.updateVideos() - } - - private async updateVideos () { - if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined - - const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() - - for (const schedule of schedules) { - const videoOnly = await VideoModel.load(schedule.videoId) - const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid) - - try { - const { video, published } = await this.updateAVideo(schedule) - - if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video) - } catch (err) { - logger.error('Cannot update video', { err }) - } - - mutexReleaser() - } - } - - private async updateAVideo (schedule: MScheduleVideoUpdate) { - let oldPrivacy: VideoPrivacy - let isNewVideo: boolean - let published = false - - const video = await sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(schedule.videoId, t) - if (video.state === VideoState.TO_TRANSCODE) return null - - logger.info('Executing scheduled video update on %s.', video.uuid) - - if (schedule.privacy) { - isNewVideo = video.isNewVideo(schedule.privacy) - oldPrivacy = video.privacy - - setVideoPrivacy(video, schedule.privacy) - await video.save({ transaction: t }) - - if (oldPrivacy === VideoPrivacy.PRIVATE) { - published = true - } - } - - await schedule.destroy({ transaction: t }) - - return video - }) - - if (!video) { - return { video, published: false } - } - - await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false }) - - return { video, published } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts deleted file mode 100644 index efb957fac..000000000 --- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { synchronizeChannel } from '../sync-channel' -import { AbstractScheduler } from './abstract-scheduler' - -export class VideoChannelSyncLatestScheduler extends AbstractScheduler { - private static instance: AbstractScheduler - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL - - private constructor () { - super() - } - - protected async internalExecute () { - if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { - logger.debug('Discard channels synchronization as the feature is disabled') - return - } - - logger.info('Checking channels to synchronize') - - const channelSyncs = await VideoChannelSyncModel.listSyncs() - - for (const sync of channelSyncs) { - const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) - - logger.info( - 'Creating video import jobs for "%s" sync with external channel "%s"', - channel.Actor.preferredUsername, sync.externalChannelUrl - ) - - const onlyAfter = sync.lastSyncAt || sync.createdAt - - await synchronizeChannel({ - channel, - externalChannelUrl: sync.externalChannelUrl, - videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, - channelSync: sync, - onlyAfter - }) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/video-views-buffer-scheduler.ts b/server/lib/schedulers/video-views-buffer-scheduler.ts deleted file mode 100644 index 244a88b14..000000000 --- a/server/lib/schedulers/video-views-buffer-scheduler.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { VideoModel } from '@server/models/video/video' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { federateVideoIfNeeded } from '../activitypub/videos' -import { Redis } from '../redis' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('views') - -export class VideoViewsBufferScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.VIDEO_VIEWS_BUFFER_UPDATE - - private constructor () { - super() - } - - protected async internalExecute () { - const videoIds = await Redis.Instance.listLocalVideosViewed() - if (videoIds.length === 0) return - - for (const videoId of videoIds) { - try { - const views = await Redis.Instance.getLocalVideoViews(videoId) - await Redis.Instance.deleteLocalVideoViews(videoId) - - const video = await VideoModel.loadFull(videoId) - if (!video) { - logger.debug('Video %d does not exist anymore, skipping videos view addition.', videoId, lTags()) - continue - } - - logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid)) - - // If this is a remote video, the origin instance will send us an update - await VideoModel.incrementViews(videoId, views) - - // Send video update - video.views += views - await federateVideoIfNeeded(video, false) - } catch (err) { - logger.error('Cannot process local video views buffer of video %d.', videoId, { err, ...lTags() }) - } - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts deleted file mode 100644 index 91625ccb5..000000000 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ /dev/null @@ -1,375 +0,0 @@ -import { move } from 'fs-extra' -import { join } from 'path' -import { getServerActor } from '@server/models/application/application' -import { VideoModel } from '@server/models/video/video' -import { - MStreamingPlaylistFiles, - MVideoAccountLight, - MVideoFile, - MVideoFileVideo, - MVideoRedundancyFileVideo, - MVideoRedundancyStreamingPlaylistVideo, - MVideoRedundancyVideo, - MVideoWithAllFiles -} from '@server/types/models' -import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' -import { logger, loggerTagsFactory } from '../../helpers/logger' -import { downloadWebTorrentVideo } from '../../helpers/webtorrent' -import { CONFIG } from '../../initializers/config' -import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' -import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' -import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' -import { getOrCreateAPVideo } from '../activitypub/videos' -import { downloadPlaylistSegments } from '../hls' -import { removeVideoRedundancy } from '../redundancy' -import { generateHLSRedundancyUrl, generateWebVideoRedundancyUrl } from '../video-urls' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('redundancy') - -type CandidateToDuplicate = { - redundancy: VideosRedundancyStrategy - video: MVideoWithAllFiles - files: MVideoFile[] - streamingPlaylists: MStreamingPlaylistFiles[] -} - -function isMVideoRedundancyFileVideo ( - o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo -): o is MVideoRedundancyFileVideo { - return !!(o as MVideoRedundancyFileVideo).VideoFile -} - -export class VideosRedundancyScheduler extends AbstractScheduler { - - private static instance: VideosRedundancyScheduler - - protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL - - private constructor () { - super() - } - - async createManualRedundancy (videoId: number) { - const videoToDuplicate = await VideoModel.loadWithFiles(videoId) - - if (!videoToDuplicate) { - logger.warn('Video to manually duplicate %d does not exist anymore.', videoId, lTags()) - return - } - - return this.createVideoRedundancies({ - video: videoToDuplicate, - redundancy: null, - files: videoToDuplicate.VideoFiles, - streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists - }) - } - - protected async internalExecute () { - for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { - logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy, lTags()) - - try { - const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) - if (!videoToDuplicate) continue - - const candidateToDuplicate = { - video: videoToDuplicate, - redundancy: redundancyConfig, - files: videoToDuplicate.VideoFiles, - streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists - } - - await this.purgeCacheIfNeeded(candidateToDuplicate) - - if (await this.isTooHeavy(candidateToDuplicate)) { - logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url, lTags(videoToDuplicate.uuid)) - continue - } - - logger.info( - 'Will duplicate video %s in redundancy scheduler "%s".', - videoToDuplicate.url, redundancyConfig.strategy, lTags(videoToDuplicate.uuid) - ) - - await this.createVideoRedundancies(candidateToDuplicate) - } catch (err) { - logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err, ...lTags() }) - } - } - - await this.extendsLocalExpiration() - - await this.purgeRemoteExpired() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - private async extendsLocalExpiration () { - const expired = await VideoRedundancyModel.listLocalExpired() - - for (const redundancyModel of expired) { - try { - const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - - // If the admin disabled the redundancy, remove this redundancy instead of extending it - if (!redundancyConfig) { - logger.info( - 'Destroying redundancy %s because the redundancy %s does not exist anymore.', - redundancyModel.url, redundancyModel.strategy - ) - - await removeVideoRedundancy(redundancyModel) - continue - } - - const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy) - - // If the admin decreased the cache size, remove this redundancy instead of extending it - if (totalUsed > redundancyConfig.size) { - logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) - - await removeVideoRedundancy(redundancyModel) - continue - } - - await this.extendsRedundancy(redundancyModel) - } catch (err) { - logger.error( - 'Cannot extend or remove expiration of %s video from our redundancy system.', - this.buildEntryLogId(redundancyModel), { err, ...lTags(redundancyModel.getVideoUUID()) } - ) - } - } - } - - private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) { - const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - // Redundancy strategy disabled, remove our redundancy instead of extending expiration - if (!redundancy) { - await removeVideoRedundancy(redundancyModel) - return - } - - await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) - } - - private async purgeRemoteExpired () { - const expired = await VideoRedundancyModel.listRemoteExpired() - - for (const redundancyModel of expired) { - try { - await removeVideoRedundancy(redundancyModel) - } catch (err) { - logger.error( - 'Cannot remove redundancy %s from our redundancy system.', - this.buildEntryLogId(redundancyModel), lTags(redundancyModel.getVideoUUID()) - ) - } - } - } - - private findVideoToDuplicate (cache: VideosRedundancyStrategy) { - if (cache.strategy === 'most-views') { - return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) - } - - if (cache.strategy === 'trending') { - return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) - } - - if (cache.strategy === 'recently-added') { - const minViews = cache.minViews - return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) - } - } - - private async createVideoRedundancies (data: CandidateToDuplicate) { - const video = await this.loadAndRefreshVideo(data.video.url) - - if (!video) { - logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url, lTags(data.video.uuid)) - - return - } - - for (const file of data.files) { - const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) - if (existingRedundancy) { - await this.extendsRedundancy(existingRedundancy) - - continue - } - - await this.createVideoFileRedundancy(data.redundancy, video, file) - } - - for (const streamingPlaylist of data.streamingPlaylists) { - const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) - if (existingRedundancy) { - await this.extendsRedundancy(existingRedundancy) - - continue - } - - await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) - } - } - - private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { - let strategy = 'manual' - let expiresOn: Date = null - - if (redundancy) { - strategy = redundancy.strategy - expiresOn = this.buildNewExpiration(redundancy.minLifetime) - } - - const file = fileArg as MVideoFileVideo - file.Video = video - - const serverActor = await getServerActor() - - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) - - const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT) - - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) - await move(tmpPath, destPath, { overwrite: true }) - - const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ - expiresOn, - url: getLocalVideoCacheFileActivityPubUrl(file), - fileUrl: generateWebVideoRedundancyUrl(file), - strategy, - videoFileId: file.id, - actorId: serverActor.id - }) - - createdModel.VideoFile = file - - await sendCreateCacheFile(serverActor, video, createdModel) - - logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url, lTags(video.uuid)) - } - - private async createStreamingPlaylistRedundancy ( - redundancy: VideosRedundancyStrategy, - video: MVideoAccountLight, - playlistArg: MStreamingPlaylistFiles - ) { - let strategy = 'manual' - let expiresOn: Date = null - - if (redundancy) { - strategy = redundancy.strategy - expiresOn = this.buildNewExpiration(redundancy.minLifetime) - } - - const playlist = Object.assign(playlistArg, { Video: video }) - const serverActor = await getServerActor() - - logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) - - const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) - const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) - - const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 - const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance - await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB) - - const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ - expiresOn, - url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), - fileUrl: generateHLSRedundancyUrl(video, playlistArg), - strategy, - videoStreamingPlaylistId: playlist.id, - actorId: serverActor.id - }) - - createdModel.VideoStreamingPlaylist = playlist - - await sendCreateCacheFile(serverActor, video, createdModel) - - logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url, lTags(video.uuid)) - } - - private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { - logger.info('Extending expiration of %s.', redundancy.url, lTags(redundancy.getVideoUUID())) - - const serverActor = await getServerActor() - - redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs) - await redundancy.save() - - await sendUpdateCacheFile(serverActor, redundancy) - } - - private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { - while (await this.isTooHeavy(candidateToDuplicate)) { - const redundancy = candidateToDuplicate.redundancy - const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) - if (!toDelete) return - - const videoId = toDelete.VideoFile - ? toDelete.VideoFile.videoId - : toDelete.VideoStreamingPlaylist.videoId - - const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId) - - for (const redundancy of redundancies) { - await removeVideoRedundancy(redundancy) - } - } - } - - private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { - const maxSize = candidateToDuplicate.redundancy.size - - const { totalUsed: alreadyUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) - - const videoSize = this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) - const willUse = alreadyUsed + videoSize - - logger.debug('Checking candidate size.', { maxSize, alreadyUsed, videoSize, willUse, ...lTags(candidateToDuplicate.video.uuid) }) - - return willUse > maxSize - } - - private buildNewExpiration (expiresAfterMs: number) { - return new Date(Date.now() + expiresAfterMs) - } - - private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { - if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` - - return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` - } - - private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number { - const fileReducer = (previous: number, current: MVideoFile) => previous + current.size - - let allFiles = files - for (const p of playlists) { - allFiles = allFiles.concat(p.VideoFiles) - } - - return allFiles.reduce(fileReducer, 0) - } - - private async loadAndRefreshVideo (videoUrl: string) { - // We need more attributes and check if the video still exists - const getVideoOptions = { - videoObject: videoUrl, - syncParam: { rates: false, shares: false, comments: false, refreshVideo: true }, - fetchType: 'all' as 'all' - } - const { video } = await getOrCreateAPVideo(getVideoOptions) - - return video - } -} diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts deleted file mode 100644 index 1ee4ae1b2..000000000 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { YoutubeDLCLI } from '@server/helpers/youtube-dl' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class YoutubeDlUpdateScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE - - private constructor () { - super() - } - - protected internalExecute () { - return YoutubeDLCLI.updateYoutubeDLBinary() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/search.ts b/server/lib/search.ts deleted file mode 100644 index b3363fbec..000000000 --- a/server/lib/search.ts +++ /dev/null @@ -1,49 +0,0 @@ -import express from 'express' -import { CONFIG } from '@server/initializers/config' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { getServerActor } from '@server/models/application/application' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { SearchTargetQuery } from '@shared/models' - -function isSearchIndexSearch (query: SearchTargetQuery) { - if (query.searchTarget === 'search-index') return true - - const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX - - if (searchIndexConfig.ENABLED !== true) return false - - if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true - - return false -} - -async function buildMutedForSearchIndex (res: express.Response) { - const serverActor = await getServerActor() - const accountIds = [ serverActor.Account.id ] - - if (res.locals.oauth) { - accountIds.push(res.locals.oauth.token.User.Account.id) - } - - const [ blockedHosts, blockedAccounts ] = await Promise.all([ - ServerBlocklistModel.listHostsBlockedBy(accountIds), - AccountBlocklistModel.listHandlesBlockedBy(accountIds) - ]) - - return { - blockedHosts, - blockedAccounts - } -} - -function isURISearch (search: string) { - if (!search) return false - - return search.startsWith('http://') || search.startsWith('https://') -} - -export { - isSearchIndexSearch, - buildMutedForSearchIndex, - isURISearch -} diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts deleted file mode 100644 index beb5d4d82..000000000 --- a/server/lib/server-config-manager.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { getServerCommit } from '@server/helpers/version' -import { CONFIG, isEmailEnabled } from '@server/initializers/config' -import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants' -import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup' -import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' -import { PluginModel } from '@server/models/server/plugin' -import { HTMLServerConfig, RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models' -import { Hooks } from './plugins/hooks' -import { PluginManager } from './plugins/plugin-manager' -import { getThemeOrDefault } from './plugins/theme-utils' -import { VideoTranscodingProfilesManager } from './transcoding/default-transcoding-profiles' - -/** - * - * Used to send the server config to clients (using REST/API or plugins API) - * We need a singleton class to manage config state depending on external events (to build menu entries etc) - * - */ - -class ServerConfigManager { - - private static instance: ServerConfigManager - - private serverCommit: string - - private homepageEnabled = false - - private constructor () {} - - async init () { - const instanceHomepage = await ActorCustomPageModel.loadInstanceHomepage() - - this.updateHomepageState(instanceHomepage?.content) - } - - updateHomepageState (content: string) { - this.homepageEnabled = !!content - } - - async getHTMLServerConfig (): Promise { - if (this.serverCommit === undefined) this.serverCommit = await getServerCommit() - - const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) - - return { - client: { - videos: { - miniature: { - displayAuthorAvatar: CONFIG.CLIENT.VIDEOS.MINIATURE.DISPLAY_AUTHOR_AVATAR, - preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME - }, - resumableUpload: { - maxChunkSize: CONFIG.CLIENT.VIDEOS.RESUMABLE_UPLOAD.MAX_CHUNK_SIZE - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH - } - } - }, - - defaults: { - publish: { - downloadEnabled: CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, - commentsEnabled: CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, - privacy: CONFIG.DEFAULTS.PUBLISH.PRIVACY, - licence: CONFIG.DEFAULTS.PUBLISH.LICENCE - }, - p2p: { - webapp: { - enabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED - }, - embed: { - enabled: CONFIG.DEFAULTS.P2P.EMBED.ENABLED - } - } - }, - - webadmin: { - configuration: { - edition: { - allowed: CONFIG.WEBADMIN.CONFIGURATION.EDITION.ALLOWED - } - } - }, - - instance: { - name: CONFIG.INSTANCE.NAME, - shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, - isNSFW: CONFIG.INSTANCE.IS_NSFW, - defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, - defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, - customizations: { - javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT, - css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS - } - }, - search: { - remoteUri: { - users: CONFIG.SEARCH.REMOTE_URI.USERS, - anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS - }, - searchIndex: { - enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, - url: CONFIG.SEARCH.SEARCH_INDEX.URL, - disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, - isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH - } - }, - plugin: { - registered: this.getRegisteredPlugins(), - registeredExternalAuths: this.getExternalAuthsPlugins(), - registeredIdAndPassAuths: this.getIdAndPassAuthPlugins() - }, - theme: { - registered: this.getRegisteredThemes(), - default: defaultTheme - }, - email: { - enabled: isEmailEnabled() - }, - contactForm: { - enabled: CONFIG.CONTACT_FORM.ENABLED - }, - serverVersion: PEERTUBE_VERSION, - serverCommit: this.serverCommit, - transcoding: { - remoteRunners: { - enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED - }, - hls: { - enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED - }, - web_videos: { - enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED - }, - enabledResolutions: this.getEnabledResolutions('vod'), - profile: CONFIG.TRANSCODING.PROFILE, - availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod') - }, - live: { - enabled: CONFIG.LIVE.ENABLED, - - allowReplay: CONFIG.LIVE.ALLOW_REPLAY, - latencySetting: { - enabled: CONFIG.LIVE.LATENCY_SETTING.ENABLED - }, - - maxDuration: CONFIG.LIVE.MAX_DURATION, - maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, - maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, - - transcoding: { - enabled: CONFIG.LIVE.TRANSCODING.ENABLED, - remoteRunners: { - enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED - }, - enabledResolutions: this.getEnabledResolutions('live'), - profile: CONFIG.LIVE.TRANSCODING.PROFILE, - availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') - }, - - rtmp: { - port: CONFIG.LIVE.RTMP.PORT - } - }, - videoStudio: { - enabled: CONFIG.VIDEO_STUDIO.ENABLED, - remoteRunners: { - enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED - } - }, - videoFile: { - update: { - enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED - } - }, - import: { - videos: { - http: { - enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED - }, - torrent: { - enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED - } - }, - videoChannelSynchronization: { - enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED - } - } - }, - avatar: { - file: { - size: { - max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME - } - }, - banner: { - file: { - size: { - max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME - } - }, - video: { - image: { - extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, - size: { - max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max - } - }, - file: { - extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME - } - }, - videoCaption: { - file: { - size: { - max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME - } - }, - user: { - videoQuota: CONFIG.USER.VIDEO_QUOTA, - videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY - }, - videoChannels: { - maxPerUser: CONFIG.VIDEO_CHANNELS.MAX_PER_USER - }, - trending: { - videos: { - intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS, - algorithms: { - enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, - default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT - } - } - }, - tracker: { - enabled: CONFIG.TRACKER.ENABLED - }, - - followings: { - instance: { - autoFollowIndex: { - indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL - } - } - }, - - broadcastMessage: { - enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, - message: CONFIG.BROADCAST_MESSAGE.MESSAGE, - level: CONFIG.BROADCAST_MESSAGE.LEVEL, - dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE - }, - - homepage: { - enabled: this.homepageEnabled - } - } - } - - async getServerConfig (ip?: string): Promise { - const { allowed } = await Hooks.wrapPromiseFun( - isSignupAllowed, - - { - ip, - signupMode: CONFIG.SIGNUP.REQUIRES_APPROVAL - ? 'request-registration' - : 'direct-registration' - }, - - CONFIG.SIGNUP.REQUIRES_APPROVAL - ? 'filter:api.user.request-signup.allowed.result' - : 'filter:api.user.signup.allowed.result' - ) - - const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip) - - const signup = { - allowed, - allowedForCurrentIP, - minimumAge: CONFIG.SIGNUP.MINIMUM_AGE, - requiresApproval: CONFIG.SIGNUP.REQUIRES_APPROVAL, - requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION - } - - const htmlConfig = await this.getHTMLServerConfig() - - return { ...htmlConfig, signup } - } - - getRegisteredThemes () { - return PluginManager.Instance.getRegisteredThemes() - .map(t => ({ - npmName: PluginModel.buildNpmName(t.name, t.type), - name: t.name, - version: t.version, - description: t.description, - css: t.css, - clientScripts: t.clientScripts - })) - } - - getRegisteredPlugins () { - return PluginManager.Instance.getRegisteredPlugins() - .map(p => ({ - npmName: PluginModel.buildNpmName(p.name, p.type), - name: p.name, - version: p.version, - description: p.description, - clientScripts: p.clientScripts - })) - } - - getEnabledResolutions (type: 'vod' | 'live') { - const transcoding = type === 'vod' - ? CONFIG.TRANSCODING - : CONFIG.LIVE.TRANSCODING - - return Object.keys(transcoding.RESOLUTIONS) - .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true) - .map(r => parseInt(r, 10)) - } - - private getIdAndPassAuthPlugins () { - const result: RegisteredIdAndPassAuthConfig[] = [] - - for (const p of PluginManager.Instance.getIdAndPassAuths()) { - for (const auth of p.idAndPassAuths) { - result.push({ - npmName: p.npmName, - name: p.name, - version: p.version, - authName: auth.authName, - weight: auth.getWeight() - }) - } - } - - return result - } - - private getExternalAuthsPlugins () { - const result: RegisteredExternalAuthConfig[] = [] - - for (const p of PluginManager.Instance.getExternalAuths()) { - for (const auth of p.externalAuths) { - result.push({ - npmName: p.npmName, - name: p.name, - version: p.version, - authName: auth.authName, - authDisplayName: auth.authDisplayName() - }) - } - } - - return result - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - ServerConfigManager -} diff --git a/server/lib/signup.ts b/server/lib/signup.ts deleted file mode 100644 index 6702c22cb..000000000 --- a/server/lib/signup.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { IPv4, IPv6, parse, subnetMatch } from 'ipaddr.js' -import { CONFIG } from '../initializers/config' -import { UserModel } from '../models/user/user' - -const isCidr = require('is-cidr') - -export type SignupMode = 'direct-registration' | 'request-registration' - -async function isSignupAllowed (options: { - signupMode: SignupMode - - ip: string // For plugins - body?: any -}): Promise<{ allowed: boolean, errorMessage?: string }> { - const { signupMode } = options - - if (CONFIG.SIGNUP.ENABLED === false) { - return { allowed: false, errorMessage: 'User registration is not allowed' } - } - - if (signupMode === 'direct-registration' && CONFIG.SIGNUP.REQUIRES_APPROVAL === true) { - return { allowed: false, errorMessage: 'User registration requires approval' } - } - - // No limit and signup is enabled - if (CONFIG.SIGNUP.LIMIT === -1) { - return { allowed: true } - } - - const totalUsers = await UserModel.countTotal() - - return { allowed: totalUsers < CONFIG.SIGNUP.LIMIT, errorMessage: 'User limit is reached on this instance' } -} - -function isSignupAllowedForCurrentIP (ip: string) { - if (!ip) return false - - const addr = parse(ip) - const excludeList = [ 'blacklist' ] - let matched = '' - - // if there is a valid, non-empty whitelist, we exclude all unknown addresses too - if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { - excludeList.push('unknown') - } - - if (addr.kind() === 'ipv4') { - const addrV4 = IPv4.parse(ip) - const rangeList = { - whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr)) - .map(cidr => IPv4.parseCIDR(cidr)), - blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr)) - .map(cidr => IPv4.parseCIDR(cidr)) - } - matched = subnetMatch(addrV4, rangeList, 'unknown') - } else if (addr.kind() === 'ipv6') { - const addrV6 = IPv6.parse(ip) - const rangeList = { - whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr)) - .map(cidr => IPv6.parseCIDR(cidr)), - blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr)) - .map(cidr => IPv6.parseCIDR(cidr)) - } - matched = subnetMatch(addrV6, rangeList, 'unknown') - } - - return !excludeList.includes(matched) -} - -// --------------------------------------------------------------------------- - -export { - isSignupAllowed, - isSignupAllowedForCurrentIP -} diff --git a/server/lib/stat-manager.ts b/server/lib/stat-manager.ts deleted file mode 100644 index 0516e7f1a..000000000 --- a/server/lib/stat-manager.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { mapSeries } from 'bluebird' -import { CONFIG } from '@server/initializers/config' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@shared/models' - -class StatsManager { - - private static instance: StatsManager - - private readonly instanceStartDate = new Date() - - private readonly inboxMessages = { - processed: 0, - errors: 0, - successes: 0, - waiting: 0, - errorsPerType: this.buildAPPerType(), - successesPerType: this.buildAPPerType() - } - - private constructor () {} - - updateInboxWaiting (inboxMessagesWaiting: number) { - this.inboxMessages.waiting = inboxMessagesWaiting - } - - addInboxProcessedSuccess (type: ActivityType) { - this.inboxMessages.processed++ - this.inboxMessages.successes++ - this.inboxMessages.successesPerType[type]++ - } - - addInboxProcessedError (type: ActivityType) { - this.inboxMessages.processed++ - this.inboxMessages.errors++ - this.inboxMessages.errorsPerType[type]++ - } - - async getStats () { - const { totalLocalVideos, totalLocalVideoViews, totalVideos } = await VideoModel.getStats() - const { totalLocalVideoComments, totalVideoComments } = await VideoCommentModel.getStats() - const { totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers } = await UserModel.getStats() - const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() - const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() - const { - totalLocalVideoChannels, - totalLocalDailyActiveVideoChannels, - totalLocalWeeklyActiveVideoChannels, - totalLocalMonthlyActiveVideoChannels - } = await VideoChannelModel.getStats() - const { totalLocalPlaylists } = await VideoPlaylistModel.getStats() - - const videosRedundancyStats = await this.buildRedundancyStats() - - const data: ServerStats = { - totalUsers, - totalDailyActiveUsers, - totalWeeklyActiveUsers, - totalMonthlyActiveUsers, - - totalLocalVideos, - totalLocalVideoViews, - totalLocalVideoComments, - totalLocalVideoFilesSize, - - totalVideos, - totalVideoComments, - - totalLocalVideoChannels, - totalLocalDailyActiveVideoChannels, - totalLocalWeeklyActiveVideoChannels, - totalLocalMonthlyActiveVideoChannels, - - totalLocalPlaylists, - - totalInstanceFollowers, - totalInstanceFollowing, - - videosRedundancy: videosRedundancyStats, - - ...this.buildAPStats() - } - - return data - } - - private buildActivityPubMessagesProcessedPerSecond () { - const now = new Date() - const startedSeconds = (now.getTime() - this.instanceStartDate.getTime()) / 1000 - - return this.inboxMessages.processed / startedSeconds - } - - private buildRedundancyStats () { - const strategies = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES - .map(r => ({ - strategy: r.strategy as VideoRedundancyStrategyWithManual, - size: r.size - })) - - strategies.push({ strategy: 'manual', size: null }) - - return mapSeries(strategies, r => { - return VideoRedundancyModel.getStats(r.strategy) - .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) - }) - } - - private buildAPPerType () { - return { - Create: 0, - Update: 0, - Delete: 0, - Follow: 0, - Accept: 0, - Reject: 0, - Announce: 0, - Undo: 0, - Like: 0, - Dislike: 0, - Flag: 0, - View: 0 - } - } - - private buildAPStats () { - return { - totalActivityPubMessagesProcessed: this.inboxMessages.processed, - - totalActivityPubMessagesSuccesses: this.inboxMessages.successes, - - // Dirty, but simpler and with type checking - totalActivityPubCreateMessagesSuccesses: this.inboxMessages.successesPerType.Create, - totalActivityPubUpdateMessagesSuccesses: this.inboxMessages.successesPerType.Update, - totalActivityPubDeleteMessagesSuccesses: this.inboxMessages.successesPerType.Delete, - totalActivityPubFollowMessagesSuccesses: this.inboxMessages.successesPerType.Follow, - totalActivityPubAcceptMessagesSuccesses: this.inboxMessages.successesPerType.Accept, - totalActivityPubRejectMessagesSuccesses: this.inboxMessages.successesPerType.Reject, - totalActivityPubAnnounceMessagesSuccesses: this.inboxMessages.successesPerType.Announce, - totalActivityPubUndoMessagesSuccesses: this.inboxMessages.successesPerType.Undo, - totalActivityPubLikeMessagesSuccesses: this.inboxMessages.successesPerType.Like, - totalActivityPubDislikeMessagesSuccesses: this.inboxMessages.successesPerType.Dislike, - totalActivityPubFlagMessagesSuccesses: this.inboxMessages.successesPerType.Flag, - totalActivityPubViewMessagesSuccesses: this.inboxMessages.successesPerType.View, - - totalActivityPubCreateMessagesErrors: this.inboxMessages.errorsPerType.Create, - totalActivityPubUpdateMessagesErrors: this.inboxMessages.errorsPerType.Update, - totalActivityPubDeleteMessagesErrors: this.inboxMessages.errorsPerType.Delete, - totalActivityPubFollowMessagesErrors: this.inboxMessages.errorsPerType.Follow, - totalActivityPubAcceptMessagesErrors: this.inboxMessages.errorsPerType.Accept, - totalActivityPubRejectMessagesErrors: this.inboxMessages.errorsPerType.Reject, - totalActivityPubAnnounceMessagesErrors: this.inboxMessages.errorsPerType.Announce, - totalActivityPubUndoMessagesErrors: this.inboxMessages.errorsPerType.Undo, - totalActivityPubLikeMessagesErrors: this.inboxMessages.errorsPerType.Like, - totalActivityPubDislikeMessagesErrors: this.inboxMessages.errorsPerType.Dislike, - totalActivityPubFlagMessagesErrors: this.inboxMessages.errorsPerType.Flag, - totalActivityPubViewMessagesErrors: this.inboxMessages.errorsPerType.View, - - totalActivityPubMessagesErrors: this.inboxMessages.errors, - - activityPubMessagesProcessedPerSecond: this.buildActivityPubMessagesProcessedPerSecond(), - totalActivityPubMessagesWaiting: this.inboxMessages.waiting - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - StatsManager -} diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts deleted file mode 100644 index 3a805a943..000000000 --- a/server/lib/sync-channel.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' -import { CONFIG } from '@server/initializers/config' -import { buildYoutubeDLImport } from '@server/lib/video-pre-import' -import { UserModel } from '@server/models/user/user' -import { VideoImportModel } from '@server/models/video/video-import' -import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' -import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' -import { CreateJobArgument, JobQueue } from './job-queue' -import { ServerConfigManager } from './server-config-manager' - -export async function synchronizeChannel (options: { - channel: MChannelAccountDefault - externalChannelUrl: string - videosCountLimit: number - channelSync?: MChannelSync - onlyAfter?: Date -}) { - const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options - - if (channelSync) { - channelSync.state = VideoChannelSyncState.PROCESSING - channelSync.lastSyncAt = new Date() - await channelSync.save() - } - - try { - const user = await UserModel.loadByChannelActorId(channel.actorId) - const youtubeDL = new YoutubeDLWrapper( - externalChannelUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) - - logger.info( - 'Fetched %d candidate URLs for sync channel %s.', - targetUrls.length, channel.Actor.preferredUsername, { targetUrls } - ) - - if (targetUrls.length === 0) { - if (channelSync) { - channelSync.state = VideoChannelSyncState.SYNCED - await channelSync.save() - } - - return - } - - const children: CreateJobArgument[] = [] - - for (const targetUrl of targetUrls) { - if (await skipImport(channel, targetUrl, onlyAfter)) continue - - const { job } = await buildYoutubeDLImport({ - user, - channel, - targetUrl, - channelSync, - importDataOverride: { - privacy: VideoPrivacy.PUBLIC - } - }) - - children.push(job) - } - - // Will update the channel sync status - const parent: CreateJobArgument = { - type: 'after-video-channel-import', - payload: { - channelSyncId: channelSync?.id - } - } - - await JobQueue.Instance.createJobWithChildren(parent, children) - } catch (err) { - logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err }) - channelSync.state = VideoChannelSyncState.FAILED - await channelSync.save() - } -} - -// --------------------------------------------------------------------------- - -async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) { - if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { - logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name) - return true - } - - if (onlyAfter) { - const youtubeDL = new YoutubeDLWrapper( - targetUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - const videoInfo = await youtubeDL.getInfoForDownload() - - const onlyAfterWithoutTime = new Date(onlyAfter) - onlyAfterWithoutTime.setHours(0, 0, 0, 0) - - if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) { - return true - } - } - - return false -} diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts deleted file mode 100644 index 0b98da14f..000000000 --- a/server/lib/thumbnail.ts +++ /dev/null @@ -1,327 +0,0 @@ -import { join } from 'path' -import { ThumbnailType } from '@shared/models' -import { generateImageFilename, generateImageFromVideoFile } from '../helpers/image-utils' -import { CONFIG } from '../initializers/config' -import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' -import { ThumbnailModel } from '../models/video/thumbnail' -import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models' -import { MThumbnail } from '../types/models/video/thumbnail' -import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' -import { VideoPathManager } from './video-path-manager' -import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' - -type ImageSize = { height?: number, width?: number } - -function updateLocalPlaylistMiniatureFromExisting (options: { - inputPath: string - playlist: MVideoPlaylistThumbnail - automaticallyGenerated: boolean - keepOriginal?: boolean // default to false - size?: ImageSize -}) { - const { inputPath, playlist, automaticallyGenerated, keepOriginal = false, size } = options - const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) - const type = ThumbnailType.MINIATURE - - const thumbnailCreator = () => { - return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) - } - - return updateThumbnailFromFunction({ - thumbnailCreator, - filename, - height, - width, - type, - automaticallyGenerated, - onDisk: true, - existingThumbnail - }) -} - -function updateRemotePlaylistMiniatureFromUrl (options: { - downloadUrl: string - playlist: MVideoPlaylistThumbnail - size?: ImageSize -}) { - const { downloadUrl, playlist, size } = options - const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) - const type = ThumbnailType.MINIATURE - - // Only save the file URL if it is a remote playlist - const fileUrl = playlist.isOwned() - ? null - : downloadUrl - - const thumbnailCreator = () => { - return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) - } - - return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) -} - -function updateLocalVideoMiniatureFromExisting (options: { - inputPath: string - video: MVideoThumbnail - type: ThumbnailType - automaticallyGenerated: boolean - size?: ImageSize - keepOriginal?: boolean // default to false -}) { - const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options - - const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - - const thumbnailCreator = () => { - return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) - } - - return updateThumbnailFromFunction({ - thumbnailCreator, - filename, - height, - width, - type, - automaticallyGenerated, - existingThumbnail, - onDisk: true - }) -} - -function generateLocalVideoMiniature (options: { - video: MVideoThumbnail - videoFile: MVideoFile - type: ThumbnailType -}) { - const { video, videoFile, type } = options - - return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => { - const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) - - const thumbnailCreator = videoFile.isAudio() - ? () => processImageFromWorker({ - path: ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, - destination: outputPath, - newSize: { width, height }, - keepOriginal: true - }) - : () => generateImageFromVideoFile({ - fromPath: input, - folder: basePath, - imageName: filename, - size: { height, width } - }) - - return updateThumbnailFromFunction({ - thumbnailCreator, - filename, - height, - width, - type, - automaticallyGenerated: true, - onDisk: true, - existingThumbnail - }) - }) -} - -// --------------------------------------------------------------------------- - -function updateLocalVideoMiniatureFromUrl (options: { - downloadUrl: string - video: MVideoThumbnail - type: ThumbnailType - size?: ImageSize -}) { - const { downloadUrl, video, type, size } = options - const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - - // Only save the file URL if it is a remote video - const fileUrl = video.isOwned() - ? null - : downloadUrl - - const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) - - // Do not change the thumbnail filename if the file did not change - const filename = thumbnailUrlChanged - ? updatedFilename - : existingThumbnail.filename - - const thumbnailCreator = () => { - if (thumbnailUrlChanged) { - return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) - } - - return Promise.resolve() - } - - return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) -} - -function updateRemoteVideoThumbnail (options: { - fileUrl: string - video: MVideoThumbnail - type: ThumbnailType - size: ImageSize - onDisk: boolean -}) { - const { fileUrl, video, type, size, onDisk } = options - const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - - const thumbnail = existingThumbnail || new ThumbnailModel() - - // Do not change the thumbnail filename if the file did not change - if (hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)) { - thumbnail.filename = generatedFilename - } - - thumbnail.height = height - thumbnail.width = width - thumbnail.type = type - thumbnail.fileUrl = fileUrl - thumbnail.onDisk = onDisk - - return thumbnail -} - -// --------------------------------------------------------------------------- - -async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) { - if (video.getMiniature().automaticallyGenerated === true) { - const miniature = await generateLocalVideoMiniature({ - video, - videoFile: video.getMaxQualityFile(), - type: ThumbnailType.MINIATURE - }) - await video.addAndSaveThumbnail(miniature) - } - - if (video.getPreview().automaticallyGenerated === true) { - const preview = await generateLocalVideoMiniature({ - video, - videoFile: video.getMaxQualityFile(), - type: ThumbnailType.PREVIEW - }) - await video.addAndSaveThumbnail(preview) - } -} - -// --------------------------------------------------------------------------- - -export { - generateLocalVideoMiniature, - regenerateMiniaturesIfNeeded, - updateLocalVideoMiniatureFromUrl, - updateLocalVideoMiniatureFromExisting, - updateRemoteVideoThumbnail, - updateRemotePlaylistMiniatureFromUrl, - updateLocalPlaylistMiniatureFromExisting -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { - const existingUrl = existingThumbnail - ? existingThumbnail.fileUrl - : null - - // If the thumbnail URL did not change and has a unique filename (introduced in 3.1), avoid thumbnail processing - return !existingUrl || existingUrl !== downloadUrl || downloadUrl.endsWith(`${video.uuid}.jpg`) -} - -function buildMetadataFromPlaylist (playlist: MVideoPlaylistThumbnail, size: ImageSize) { - const filename = playlist.generateThumbnailName() - const basePath = CONFIG.STORAGE.THUMBNAILS_DIR - - return { - filename, - basePath, - existingThumbnail: playlist.Thumbnail, - outputPath: join(basePath, filename), - height: size ? size.height : THUMBNAILS_SIZE.height, - width: size ? size.width : THUMBNAILS_SIZE.width - } -} - -function buildMetadataFromVideo (video: MVideoThumbnail, type: ThumbnailType, size?: ImageSize) { - const existingThumbnail = Array.isArray(video.Thumbnails) - ? video.Thumbnails.find(t => t.type === type) - : undefined - - if (type === ThumbnailType.MINIATURE) { - const filename = generateImageFilename() - const basePath = CONFIG.STORAGE.THUMBNAILS_DIR - - return { - filename, - basePath, - existingThumbnail, - outputPath: join(basePath, filename), - height: size ? size.height : THUMBNAILS_SIZE.height, - width: size ? size.width : THUMBNAILS_SIZE.width - } - } - - if (type === ThumbnailType.PREVIEW) { - const filename = generateImageFilename() - const basePath = CONFIG.STORAGE.PREVIEWS_DIR - - return { - filename, - basePath, - existingThumbnail, - outputPath: join(basePath, filename), - height: size ? size.height : PREVIEWS_SIZE.height, - width: size ? size.width : PREVIEWS_SIZE.width - } - } - - return undefined -} - -async function updateThumbnailFromFunction (parameters: { - thumbnailCreator: () => Promise - filename: string - height: number - width: number - type: ThumbnailType - onDisk: boolean - automaticallyGenerated?: boolean - fileUrl?: string - existingThumbnail?: MThumbnail -}) { - const { - thumbnailCreator, - filename, - width, - height, - type, - existingThumbnail, - onDisk, - automaticallyGenerated = null, - fileUrl = null - } = parameters - - const oldFilename = existingThumbnail && existingThumbnail.filename !== filename - ? existingThumbnail.filename - : undefined - - const thumbnail: MThumbnail = existingThumbnail || new ThumbnailModel() - - thumbnail.filename = filename - thumbnail.height = height - thumbnail.width = width - thumbnail.type = type - thumbnail.fileUrl = fileUrl - thumbnail.automaticallyGenerated = automaticallyGenerated - thumbnail.onDisk = onDisk - - if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename - - await thumbnailCreator() - - return thumbnail -} diff --git a/server/lib/timeserie.ts b/server/lib/timeserie.ts deleted file mode 100644 index 08b12129a..000000000 --- a/server/lib/timeserie.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { logger } from '@server/helpers/logger' - -function buildGroupByAndBoundaries (startDateString: string, endDateString: string) { - const startDate = new Date(startDateString) - const endDate = new Date(endDateString) - - const groupInterval = buildGroupInterval(startDate, endDate) - - logger.debug('Found "%s" group interval.', groupInterval, { startDate, endDate }) - - // Remove parts of the date we don't need - if (groupInterval.endsWith(' month') || groupInterval.endsWith(' months')) { - startDate.setDate(1) - startDate.setHours(0, 0, 0, 0) - } else if (groupInterval.endsWith(' day') || groupInterval.endsWith(' days')) { - startDate.setHours(0, 0, 0, 0) - } else if (groupInterval.endsWith(' hour') || groupInterval.endsWith(' hours')) { - startDate.setMinutes(0, 0, 0) - } else { - startDate.setSeconds(0, 0) - } - - return { - groupInterval, - startDate, - endDate - } -} - -// --------------------------------------------------------------------------- - -export { - buildGroupByAndBoundaries -} - -// --------------------------------------------------------------------------- - -function buildGroupInterval (startDate: Date, endDate: Date): string { - const aYear = 31536000 - const aMonth = 2678400 - const aDay = 86400 - const anHour = 3600 - const aMinute = 60 - - const diffSeconds = (endDate.getTime() - startDate.getTime()) / 1000 - - if (diffSeconds >= 6 * aYear) return '6 months' - if (diffSeconds >= 2 * aYear) return '1 month' - if (diffSeconds >= 6 * aMonth) return '7 days' - if (diffSeconds >= 2 * aMonth) return '2 days' - - if (diffSeconds >= 15 * aDay) return '1 day' - if (diffSeconds >= 8 * aDay) return '12 hours' - if (diffSeconds >= 4 * aDay) return '6 hours' - - if (diffSeconds >= 15 * anHour) return '1 hour' - - if (diffSeconds >= 180 * aMinute) return '10 minutes' - - return '1 minute' -} diff --git a/server/lib/transcoding/create-transcoding-job.ts b/server/lib/transcoding/create-transcoding-job.ts deleted file mode 100644 index d78e68b87..000000000 --- a/server/lib/transcoding/create-transcoding-job.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' -import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared' - -export function createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean -}) { - return getJobBuilder().createOptimizeOrMergeAudioJobs(options) -} - -// --------------------------------------------------------------------------- - -export function createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId -}) { - return getJobBuilder().createTranscodingJobs(options) -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function getJobBuilder () { - if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) { - return new TranscodingRunnerJobBuilder() - } - - return new TranscodingJobQueueBuilder() -} diff --git a/server/lib/transcoding/default-transcoding-profiles.ts b/server/lib/transcoding/default-transcoding-profiles.ts deleted file mode 100644 index 8f8fdd026..000000000 --- a/server/lib/transcoding/default-transcoding-profiles.ts +++ /dev/null @@ -1,143 +0,0 @@ - -import { logger } from '@server/helpers/logger' -import { FFmpegCommandWrapper, getDefaultAvailableEncoders } from '@shared/ffmpeg' -import { AvailableEncoders, EncoderOptionsBuilder } from '@shared/models' - -// --------------------------------------------------------------------------- -// Profile manager to get and change default profiles -// --------------------------------------------------------------------------- - -class VideoTranscodingProfilesManager { - private static instance: VideoTranscodingProfilesManager - - // 1 === less priority - private readonly encodersPriorities = { - vod: this.buildDefaultEncodersPriorities(), - live: this.buildDefaultEncodersPriorities() - } - - private readonly availableEncoders = getDefaultAvailableEncoders() - - private availableProfiles = { - vod: [] as string[], - live: [] as string[] - } - - private constructor () { - this.buildAvailableProfiles() - } - - getAvailableEncoders (): AvailableEncoders { - return { - available: this.availableEncoders, - encodersToTry: { - vod: { - video: this.getEncodersByPriority('vod', 'video'), - audio: this.getEncodersByPriority('vod', 'audio') - }, - live: { - video: this.getEncodersByPriority('live', 'video'), - audio: this.getEncodersByPriority('live', 'audio') - } - } - } - } - - getAvailableProfiles (type: 'vod' | 'live') { - return this.availableProfiles[type] - } - - addProfile (options: { - type: 'vod' | 'live' - encoder: string - profile: string - builder: EncoderOptionsBuilder - }) { - const { type, encoder, profile, builder } = options - - const encoders = this.availableEncoders[type] - - if (!encoders[encoder]) encoders[encoder] = {} - encoders[encoder][profile] = builder - - this.buildAvailableProfiles() - } - - removeProfile (options: { - type: 'vod' | 'live' - encoder: string - profile: string - }) { - const { type, encoder, profile } = options - - delete this.availableEncoders[type][encoder][profile] - this.buildAvailableProfiles() - } - - addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { - this.encodersPriorities[type][streamType].push({ name: encoder, priority }) - - FFmpegCommandWrapper.resetSupportedEncoders() - } - - removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { - this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType] - .filter(o => o.name !== encoder && o.priority !== priority) - - FFmpegCommandWrapper.resetSupportedEncoders() - } - - private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') { - return this.encodersPriorities[type][streamType] - .sort((e1, e2) => { - if (e1.priority > e2.priority) return -1 - else if (e1.priority === e2.priority) return 0 - - return 1 - }) - .map(e => e.name) - } - - private buildAvailableProfiles () { - for (const type of [ 'vod', 'live' ]) { - const result = new Set() - - const encoders = this.availableEncoders[type] - - for (const encoderName of Object.keys(encoders)) { - for (const profile of Object.keys(encoders[encoderName])) { - result.add(profile) - } - } - - this.availableProfiles[type] = Array.from(result) - } - - logger.debug('Available transcoding profiles built.', { availableProfiles: this.availableProfiles }) - } - - private buildDefaultEncodersPriorities () { - return { - video: [ - { name: 'libx264', priority: 100 } - ], - - // Try the first one, if not available try the second one etc - audio: [ - // we favor VBR, if a good AAC encoder is available - { name: 'libfdk_aac', priority: 200 }, - { name: 'aac', priority: 100 } - ] - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - VideoTranscodingProfilesManager -} diff --git a/server/lib/transcoding/ended-transcoding.ts b/server/lib/transcoding/ended-transcoding.ts deleted file mode 100644 index d31674ede..000000000 --- a/server/lib/transcoding/ended-transcoding.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { moveToNextState } from '../video-state' - -export async function onTranscodingEnded (options: { - video: MVideo - isNewVideo: boolean - moveVideoToNextState: boolean -}) { - const { video, isNewVideo, moveVideoToNextState } = options - - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - if (moveVideoToNextState) { - await retryTransactionWrapper(moveToNextState, { video, isNewVideo }) - } -} diff --git a/server/lib/transcoding/hls-transcoding.ts b/server/lib/transcoding/hls-transcoding.ts deleted file mode 100644 index 2c325d9ee..000000000 --- a/server/lib/transcoding/hls-transcoding.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { MutexInterface } from 'async-mutex' -import { Job } from 'bullmq' -import { ensureDir, move, stat } from 'fs-extra' -import { basename, extname as extnameUtil, join } from 'path' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { sequelizeTypescript } from '@server/initializers/database' -import { MVideo, MVideoFile } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' -import { VideoResolution } from '@shared/models' -import { CONFIG } from '../../initializers/config' -import { VideoFileModel } from '../../models/video/video-file' -import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' -import { updatePlaylistAfterFileChange } from '../hls' -import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' -import { buildFileMetadata } from '../video-file' -import { VideoPathManager } from '../video-path-manager' -import { buildFFmpegVOD } from './shared' - -// Concat TS segments from a live video to a fragmented mp4 HLS playlist -export async function generateHlsPlaylistResolutionFromTS (options: { - video: MVideo - concatenatedTsFilePath: string - resolution: VideoResolution - fps: number - isAAC: boolean - inputFileMutexReleaser: MutexInterface.Releaser -}) { - return generateHlsPlaylistCommon({ - type: 'hls-from-ts' as 'hls-from-ts', - inputPath: options.concatenatedTsFilePath, - - ...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ]) - }) -} - -// Generate an HLS playlist from an input file, and update the master playlist -export function generateHlsPlaylistResolution (options: { - video: MVideo - videoInputPath: string - resolution: VideoResolution - fps: number - copyCodecs: boolean - inputFileMutexReleaser: MutexInterface.Releaser - job?: Job -}) { - return generateHlsPlaylistCommon({ - type: 'hls' as 'hls', - inputPath: options.videoInputPath, - - ...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) - }) -} - -export async function onHLSVideoFileTranscoding (options: { - video: MVideo - videoFile: MVideoFile - videoOutputPath: string - m3u8OutputPath: string -}) { - const { video, videoFile, videoOutputPath, m3u8OutputPath } = options - - // Create or update the playlist - const playlist = await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) - }) - }) - videoFile.videoStreamingPlaylistId = playlist.id - - const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile) - await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) - - // Move playlist file - const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath)) - await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true }) - // Move video file - await move(videoOutputPath, videoFilePath, { overwrite: true }) - - // Update video duration if it was not set (in case of a live for example) - if (!video.duration) { - video.duration = await getVideoStreamDuration(videoFilePath) - await video.save() - } - - const stats = await stat(videoFilePath) - - videoFile.size = stats.size - videoFile.fps = await getVideoStreamFPS(videoFilePath) - videoFile.metadata = await buildFileMetadata(videoFilePath) - - await createTorrentAndSetInfoHash(playlist, videoFile) - - const oldFile = await VideoFileModel.loadHLSFile({ - playlistId: playlist.id, - fps: videoFile.fps, - resolution: videoFile.resolution - }) - - if (oldFile) { - await video.removeStreamingPlaylistVideoFile(playlist, oldFile) - await oldFile.destroy() - } - - const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined) - - await updatePlaylistAfterFileChange(video, playlist) - - return { resolutionPlaylistPath, videoFile: savedVideoFile } - } finally { - mutexReleaser() - } -} - -// --------------------------------------------------------------------------- - -async function generateHlsPlaylistCommon (options: { - type: 'hls' | 'hls-from-ts' - video: MVideo - inputPath: string - - resolution: VideoResolution - fps: number - - inputFileMutexReleaser: MutexInterface.Releaser - - copyCodecs?: boolean - isAAC?: boolean - - job?: Job -}) { - const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - - const videoTranscodedBasePath = join(transcodeDirectory, type) - await ensureDir(videoTranscodedBasePath) - - const videoFilename = generateHLSVideoFilename(resolution) - const videoOutputPath = join(videoTranscodedBasePath, videoFilename) - - const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) - const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename) - - const transcodeOptions = { - type, - - inputPath, - outputPath: m3u8OutputPath, - - resolution, - fps, - copyCodecs, - - isAAC, - - inputFileMutexReleaser, - - hlsPlaylist: { - videoFilename - } - } - - await buildFFmpegVOD(job).transcode(transcodeOptions) - - const newVideoFile = new VideoFileModel({ - resolution, - extname: extnameUtil(videoFilename), - size: 0, - filename: videoFilename, - fps: -1 - }) - - await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath }) -} diff --git a/server/lib/transcoding/shared/ffmpeg-builder.ts b/server/lib/transcoding/shared/ffmpeg-builder.ts deleted file mode 100644 index 441445ec4..000000000 --- a/server/lib/transcoding/shared/ffmpeg-builder.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Job } from 'bullmq' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { logger } from '@server/helpers/logger' -import { FFmpegVOD } from '@shared/ffmpeg' -import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles' - -export function buildFFmpegVOD (job?: Job) { - return new FFmpegVOD({ - ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()), - - updateJobProgress: progress => { - if (!job) return - - job.updateProgress(progress) - .catch(err => logger.error('Cannot update ffmpeg job progress', { err })) - } - }) -} diff --git a/server/lib/transcoding/shared/index.ts b/server/lib/transcoding/shared/index.ts deleted file mode 100644 index f0b45bcbb..000000000 --- a/server/lib/transcoding/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './job-builders' -export * from './ffmpeg-builder' diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts deleted file mode 100644 index 15fc814ae..000000000 --- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts +++ /dev/null @@ -1,21 +0,0 @@ - -import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' - -export abstract class AbstractJobBuilder { - - abstract createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean - }): Promise - - abstract createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId | null - }): Promise -} diff --git a/server/lib/transcoding/shared/job-builders/index.ts b/server/lib/transcoding/shared/job-builders/index.ts deleted file mode 100644 index 9b1c82adf..000000000 --- a/server/lib/transcoding/shared/job-builders/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './transcoding-job-queue-builder' -export * from './transcoding-runner-job-builder' diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts deleted file mode 100644 index 0505c2b2f..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts +++ /dev/null @@ -1,322 +0,0 @@ -import Bluebird from 'bluebird' -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' -import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' -import { Hooks } from '@server/lib/plugins/hooks' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' -import { - HLSTranscodingPayload, - MergeAudioTranscodingPayload, - NewWebVideoResolutionTranscodingPayload, - OptimizeTranscodingPayload, - VideoTranscodingPayload -} from '@shared/models' -import { getTranscodingJobPriority } from '../../transcoding-priority' -import { canDoQuickTranscode } from '../../transcoding-quick-transcode' -import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions' -import { AbstractJobBuilder } from './abstract-job-builder' - -export class TranscodingJobQueueBuilder extends AbstractJobBuilder { - - async createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean - }) { - const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options - - let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload - let nextTranscodingSequentialJobPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] - - const mutexReleaser = videoFileAlreadyLocked - ? () => {} - : await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await videoFile.reload() - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { - const probe = await ffprobePromise(videoFilePath) - - const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) - const hasAudio = await hasAudioStream(videoFilePath, probe) - const quickTranscode = await canDoQuickTranscode(videoFilePath, probe) - const inputFPS = videoFile.isAudio() - ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value - : await getVideoStreamFPS(videoFilePath, probe) - - const maxResolution = await isAudioFile(videoFilePath, probe) - ? DEFAULT_AUDIO_RESOLUTION - : buildOriginalFileResolution(resolution) - - if (CONFIG.TRANSCODING.HLS.ENABLED === true) { - nextTranscodingSequentialJobPayloads.push([ - this.buildHLSJobPayload({ - deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, - - // We had some issues with a web video quick transcoded while producing a HLS version of it - copyCodecs: !quickTranscode, - - resolution: maxResolution, - fps: computeOutputFPS({ inputFPS, resolution: maxResolution }), - videoUUID: video.uuid, - isNewVideo - }) - ]) - } - - const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({ - video, - inputVideoResolution: maxResolution, - inputVideoFPS: inputFPS, - hasAudio, - isNewVideo - }) - - nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ] - - const hasChildren = nextTranscodingSequentialJobPayloads.length !== 0 - mergeOrOptimizePayload = videoFile.isAudio() - ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo, hasChildren }) - : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode, hasChildren }) - }) - } finally { - mutexReleaser() - } - - const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => { - return Bluebird.mapSeries(payloads, payload => { - return this.buildTranscodingJob({ payload, user }) - }) - }) - - const transcodingJobBuilderJob: CreateJobArgument = { - type: 'transcoding-job-builder', - payload: { - videoUUID: video.uuid, - sequentialJobs: nextTranscodingSequentialJobs - } - } - - const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user }) - - await JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ]) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - } - - // --------------------------------------------------------------------------- - - async createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId | null - }) { - const { video, transcodingType, resolutions, isNewVideo } = options - - const maxResolution = Math.max(...resolutions) - const childrenResolutions = resolutions.filter(r => r !== maxResolution) - - logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) - - const { fps: inputFPS } = await video.probeMaxQualityFile() - - const children = childrenResolutions.map(resolution => { - const fps = computeOutputFPS({ inputFPS, resolution }) - - if (transcodingType === 'hls') { - return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) - } - - if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { - return this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) - } - - throw new Error('Unknown transcoding type') - }) - - const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) - - const parent = transcodingType === 'hls' - ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) - : this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) - - // Process the last resolution after the other ones to prevent concurrency issue - // Because low resolutions use the biggest one as ffmpeg input - await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null }) - } - - // --------------------------------------------------------------------------- - - private async createTranscodingJobsWithChildren (options: { - videoUUID: string - parent: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload) - children: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)[] - user: MUserId | null - }) { - const { videoUUID, parent, children, user } = options - - const parentJob = await this.buildTranscodingJob({ payload: parent, user }) - const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user })) - - await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs) - - await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length) - } - - private async buildTranscodingJob (options: { - payload: VideoTranscodingPayload - user: MUserId | null // null means we don't want priority - }) { - const { user, payload } = options - - return { - type: 'video-transcoding' as 'video-transcoding', - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }), - payload - } - } - - private async buildLowerResolutionJobPayloads (options: { - video: MVideoWithFileThumbnail - inputVideoResolution: number - inputVideoFPS: number - hasAudio: boolean - isNewVideo: boolean - }) { - const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options - - // Create transcoding jobs if there are enabled resolutions - const resolutionsEnabled = await Hooks.wrapObject( - computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), - 'filter:transcoding.auto.resolutions-to-transcode.result', - options - ) - - const sequentialPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] - - for (const resolution of resolutionsEnabled) { - const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) - - if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { - const payloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ - this.buildWebVideoJobPayload({ - videoUUID: video.uuid, - resolution, - fps, - isNewVideo - }) - ] - - // Create a subsequent job to create HLS resolution that will just copy web video codecs - if (CONFIG.TRANSCODING.HLS.ENABLED) { - payloads.push( - this.buildHLSJobPayload({ - videoUUID: video.uuid, - resolution, - fps, - isNewVideo, - copyCodecs: true - }) - ) - } - - sequentialPayloads.push(payloads) - } else if (CONFIG.TRANSCODING.HLS.ENABLED) { - sequentialPayloads.push([ - this.buildHLSJobPayload({ - videoUUID: video.uuid, - resolution, - fps, - copyCodecs: false, - isNewVideo - }) - ]) - } - } - - return sequentialPayloads - } - - private buildHLSJobPayload (options: { - videoUUID: string - resolution: number - fps: number - isNewVideo: boolean - deleteWebVideoFiles?: boolean // default false - copyCodecs?: boolean // default false - }): HLSTranscodingPayload { - const { videoUUID, resolution, fps, isNewVideo, deleteWebVideoFiles = false, copyCodecs = false } = options - - return { - type: 'new-resolution-to-hls', - videoUUID, - resolution, - fps, - copyCodecs, - isNewVideo, - deleteWebVideoFiles - } - } - - private buildWebVideoJobPayload (options: { - videoUUID: string - resolution: number - fps: number - isNewVideo: boolean - }): NewWebVideoResolutionTranscodingPayload { - const { videoUUID, resolution, fps, isNewVideo } = options - - return { - type: 'new-resolution-to-web-video', - videoUUID, - isNewVideo, - resolution, - fps - } - } - - private buildMergeAudioPayload (options: { - videoUUID: string - isNewVideo: boolean - hasChildren: boolean - }): MergeAudioTranscodingPayload { - const { videoUUID, isNewVideo, hasChildren } = options - - return { - type: 'merge-audio-to-web-video', - resolution: DEFAULT_AUDIO_RESOLUTION, - fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, - videoUUID, - isNewVideo, - hasChildren - } - } - - private buildOptimizePayload (options: { - videoUUID: string - quickTranscode: boolean - isNewVideo: boolean - hasChildren: boolean - }): OptimizeTranscodingPayload { - const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options - - return { - type: 'optimize-to-web-video', - videoUUID, - isNewVideo, - hasChildren, - quickTranscode - } - } -} diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts deleted file mode 100644 index f0671bd7a..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' -import { Hooks } from '@server/lib/plugins/hooks' -import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' -import { getTranscodingJobPriority } from '../../transcoding-priority' -import { computeResolutionsToTranscode } from '../../transcoding-resolutions' -import { AbstractJobBuilder } from './abstract-job-builder' - -/** - * - * Class to build transcoding job in the local job queue - * - */ - -const lTags = loggerTagsFactory('transcoding') - -export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { - - async createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean - }) { - const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options - - const mutexReleaser = videoFileAlreadyLocked - ? () => {} - : await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await videoFile.reload() - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { - const probe = await ffprobePromise(videoFilePath) - - const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) - const hasAudio = await hasAudioStream(videoFilePath, probe) - const inputFPS = videoFile.isAudio() - ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value - : await getVideoStreamFPS(videoFilePath, probe) - - const maxResolution = await isAudioFile(videoFilePath, probe) - ? DEFAULT_AUDIO_RESOLUTION - : resolution - - const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - - const mainRunnerJob = videoFile.isAudio() - ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) - - if (CONFIG.TRANSCODING.HLS.ENABLED === true) { - await new VODHLSTranscodingJobHandler().create({ - video, - deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, - resolution: maxResolution, - fps, - isNewVideo, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - - await this.buildLowerResolutionJobPayloads({ - video, - inputVideoResolution: maxResolution, - inputVideoFPS: inputFPS, - hasAudio, - isNewVideo, - mainRunnerJob, - user - }) - }) - } finally { - mutexReleaser() - } - } - - // --------------------------------------------------------------------------- - - async createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId | null - }) { - const { video, transcodingType, resolutions, isNewVideo, user } = options - - const maxResolution = Math.max(...resolutions) - const { fps: inputFPS } = await video.probeMaxQualityFile() - const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - - const childrenResolutions = resolutions.filter(r => r !== maxResolution) - - logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) - - // Process the last resolution before the other ones to prevent concurrency issue - // Because low resolutions use the biggest one as ffmpeg input - const mainJob = transcodingType === 'hls' - // eslint-disable-next-line max-len - ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) - - for (const resolution of childrenResolutions) { - const dependsOnRunnerJob = mainJob - const fps = computeOutputFPS({ inputFPS, resolution }) - - if (transcodingType === 'hls') { - await new VODHLSTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - deleteWebVideoFiles: false, - dependsOnRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - continue - } - - if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { - await new VODWebVideoTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - dependsOnRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - continue - } - - throw new Error('Unknown transcoding type') - } - } - - private async buildLowerResolutionJobPayloads (options: { - mainRunnerJob: MRunnerJob - video: MVideoWithFileThumbnail - inputVideoResolution: number - inputVideoFPS: number - hasAudio: boolean - isNewVideo: boolean - user: MUserId - }) { - const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options - - // Create transcoding jobs if there are enabled resolutions - const resolutionsEnabled = await Hooks.wrapObject( - computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), - 'filter:transcoding.auto.resolutions-to-transcode.result', - options - ) - - logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) }) - - for (const resolution of resolutionsEnabled) { - const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) - - if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { - await new VODWebVideoTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - - if (CONFIG.TRANSCODING.HLS.ENABLED) { - await new VODHLSTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - deleteWebVideoFiles: false, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - } - } -} diff --git a/server/lib/transcoding/transcoding-priority.ts b/server/lib/transcoding/transcoding-priority.ts deleted file mode 100644 index 82ab6f2f1..000000000 --- a/server/lib/transcoding/transcoding-priority.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { JOB_PRIORITY } from '@server/initializers/constants' -import { VideoModel } from '@server/models/video/video' -import { MUserId } from '@server/types/models' - -export async function getTranscodingJobPriority (options: { - user: MUserId - fallback: number - type: 'vod' | 'studio' -}) { - const { user, fallback, type } = options - - if (!user) return fallback - - const now = new Date() - const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) - - const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) - - const base = type === 'vod' - ? JOB_PRIORITY.TRANSCODING - : JOB_PRIORITY.VIDEO_STUDIO - - return base + videoUploadedByUser -} diff --git a/server/lib/transcoding/transcoding-quick-transcode.ts b/server/lib/transcoding/transcoding-quick-transcode.ts deleted file mode 100644 index 53f12cd06..000000000 --- a/server/lib/transcoding/transcoding-quick-transcode.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { CONFIG } from '@server/initializers/config' -import { canDoQuickAudioTranscode, canDoQuickVideoTranscode, ffprobePromise } from '@shared/ffmpeg' - -export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise { - if (CONFIG.TRANSCODING.PROFILE !== 'default') return false - - const probe = existingProbe || await ffprobePromise(path) - - return await canDoQuickVideoTranscode(path, probe) && - await canDoQuickAudioTranscode(path, probe) -} diff --git a/server/lib/transcoding/transcoding-resolutions.ts b/server/lib/transcoding/transcoding-resolutions.ts deleted file mode 100644 index 9a6bf5722..000000000 --- a/server/lib/transcoding/transcoding-resolutions.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { toEven } from '@shared/core-utils' -import { VideoResolution } from '@shared/models' - -export function buildOriginalFileResolution (inputResolution: number) { - if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { - return toEven(inputResolution) - } - - const resolutions = computeResolutionsToTranscode({ - input: inputResolution, - type: 'vod', - includeInput: false, - strictLower: false, - // We don't really care about the audio resolution in this context - hasAudio: true - }) - - if (resolutions.length === 0) { - return toEven(inputResolution) - } - - return Math.max(...resolutions) -} - -export function computeResolutionsToTranscode (options: { - input: number - type: 'vod' | 'live' - includeInput: boolean - strictLower: boolean - hasAudio: boolean -}) { - const { input, type, includeInput, strictLower, hasAudio } = options - - const configResolutions = type === 'vod' - ? CONFIG.TRANSCODING.RESOLUTIONS - : CONFIG.LIVE.TRANSCODING.RESOLUTIONS - - const resolutionsEnabled = new Set() - - // Put in the order we want to proceed jobs - const availableResolutions: VideoResolution[] = [ - VideoResolution.H_NOVIDEO, - VideoResolution.H_480P, - VideoResolution.H_360P, - VideoResolution.H_720P, - VideoResolution.H_240P, - VideoResolution.H_144P, - VideoResolution.H_1080P, - VideoResolution.H_1440P, - VideoResolution.H_4K - ] - - for (const resolution of availableResolutions) { - // Resolution not enabled - if (configResolutions[resolution + 'p'] !== true) continue - // Too big resolution for input file - if (input < resolution) continue - // We only want lower resolutions than input file - if (strictLower && input === resolution) continue - // Audio resolutio but no audio in the video - if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue - - resolutionsEnabled.add(resolution) - } - - if (includeInput) { - // Always use an even resolution to avoid issues with ffmpeg - resolutionsEnabled.add(toEven(input)) - } - - return Array.from(resolutionsEnabled) -} diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts deleted file mode 100644 index f92d457a0..000000000 --- a/server/lib/transcoding/web-transcoding.ts +++ /dev/null @@ -1,263 +0,0 @@ -import { Job } from 'bullmq' -import { copyFile, move, remove, stat } from 'fs-extra' -import { basename, join } from 'path' -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { VideoModel } from '@server/models/video/video' -import { MVideoFile, MVideoFullLight } from '@server/types/models' -import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@shared/ffmpeg' -import { VideoResolution, VideoStorage } from '@shared/models' -import { CONFIG } from '../../initializers/config' -import { VideoFileModel } from '../../models/video/video-file' -import { JobQueue } from '../job-queue' -import { generateWebVideoFilename } from '../paths' -import { buildFileMetadata } from '../video-file' -import { VideoPathManager } from '../video-path-manager' -import { buildFFmpegVOD } from './shared' -import { buildOriginalFileResolution } from './transcoding-resolutions' - -// Optimize the original video file and replace it. The resolution is not changed. -export async function optimizeOriginalVideofile (options: { - video: MVideoFullLight - inputVideoFile: MVideoFile - quickTranscode: boolean - job: Job -}) { - const { video, inputVideoFile, quickTranscode, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - // Will be released by our transcodeVOD function once ffmpeg is ran - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await inputVideoFile.reload() - - const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { - const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) - - const transcodeType: TranscodeVODOptionsType = quickTranscode - ? 'quick-transcode' - : 'video' - - const resolution = buildOriginalFileResolution(inputVideoFile.resolution) - const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution }) - - // Could be very long! - await buildFFmpegVOD(job).transcode({ - type: transcodeType, - - inputPath: videoInputPath, - outputPath: videoOutputPath, - - inputFileMutexReleaser, - - resolution, - fps - }) - - // Important to do this before getVideoFilename() to take in account the new filename - inputVideoFile.resolution = resolution - inputVideoFile.extname = newExtname - inputVideoFile.filename = generateWebVideoFilename(resolution, newExtname) - inputVideoFile.storage = VideoStorage.FILE_SYSTEM - - const { videoFile } = await onWebVideoFileTranscoding({ - video, - videoFile: inputVideoFile, - videoOutputPath - }) - - await remove(videoInputPath) - - return { transcodeType, videoFile } - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -// Transcode the original video file to a lower resolution compatible with web browsers -export async function transcodeNewWebVideoResolution (options: { - video: MVideoFullLight - resolution: VideoResolution - fps: number - job: Job -}) { - const { video: videoArg, resolution, fps, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) - - try { - const video = await VideoModel.loadFull(videoArg.uuid) - const file = video.getMaxQualityFile().withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { - const newVideoFile = new VideoFileModel({ - resolution, - extname: newExtname, - filename: generateWebVideoFilename(resolution, newExtname), - size: 0, - videoId: video.id - }) - - const videoOutputPath = join(transcodeDirectory, newVideoFile.filename) - - const transcodeOptions = { - type: 'video' as 'video', - - inputPath: videoInputPath, - outputPath: videoOutputPath, - - inputFileMutexReleaser, - - resolution, - fps - } - - await buildFFmpegVOD(job).transcode(transcodeOptions) - - return onWebVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -// Merge an image with an audio file to create a video -export async function mergeAudioVideofile (options: { - video: MVideoFullLight - resolution: VideoResolution - fps: number - job: Job -}) { - const { video: videoArg, resolution, fps, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) - - try { - const video = await VideoModel.loadFull(videoArg.uuid) - const inputVideoFile = video.getMinQualityFile() - - const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { - const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) - - // If the user updates the video preview during transcoding - const previewPath = video.getPreview().getPath() - const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) - await copyFile(previewPath, tmpPreviewPath) - - const transcodeOptions = { - type: 'merge-audio' as 'merge-audio', - - inputPath: tmpPreviewPath, - outputPath: videoOutputPath, - - inputFileMutexReleaser, - - audioPath: audioInputPath, - resolution, - fps - } - - try { - await buildFFmpegVOD(job).transcode(transcodeOptions) - - await remove(audioInputPath) - await remove(tmpPreviewPath) - } catch (err) { - await remove(tmpPreviewPath) - throw err - } - - // Important to do this before getVideoFilename() to take in account the new file extension - inputVideoFile.extname = newExtname - inputVideoFile.resolution = resolution - inputVideoFile.filename = generateWebVideoFilename(inputVideoFile.resolution, newExtname) - - // ffmpeg generated a new video file, so update the video duration - // See https://trac.ffmpeg.org/ticket/5456 - video.duration = await getVideoStreamDuration(videoOutputPath) - await video.save() - - return onWebVideoFileTranscoding({ - video, - videoFile: inputVideoFile, - videoOutputPath, - wasAudioFile: true - }) - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -export async function onWebVideoFileTranscoding (options: { - video: MVideoFullLight - videoFile: MVideoFile - videoOutputPath: string - wasAudioFile?: boolean // default false -}) { - const { video, videoFile, videoOutputPath, wasAudioFile } = options - - const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) - - const stats = await stat(videoOutputPath) - - const probe = await ffprobePromise(videoOutputPath) - const fps = await getVideoStreamFPS(videoOutputPath, probe) - const metadata = await buildFileMetadata(videoOutputPath, probe) - - await move(videoOutputPath, outputPath, { overwrite: true }) - - videoFile.size = stats.size - videoFile.fps = fps - videoFile.metadata = metadata - - await createTorrentAndSetInfoHash(video, videoFile) - - const oldFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) - if (oldFile) await video.removeWebVideoFile(oldFile) - - await VideoFileModel.customUpsert(videoFile, 'video', undefined) - video.VideoFiles = await video.$get('VideoFiles') - - if (wasAudioFile) { - await JobQueue.Instance.createJob({ - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - // No need to federate, we process these jobs sequentially - federate: false - } - }) - } - - return { video, videoFile } - } finally { - mutexReleaser() - } -} diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts deleted file mode 100644 index c7e0eb414..000000000 --- a/server/lib/uploadx.ts +++ /dev/null @@ -1,37 +0,0 @@ -import express from 'express' -import { buildLogger } from '@server/helpers/logger' -import { getResumableUploadPath } from '@server/helpers/upload' -import { CONFIG } from '@server/initializers/config' -import { LogLevel, Uploadx } from '@uploadx/core' -import { extname } from 'path' - -const logger = buildLogger('uploadx') - -const uploadx = new Uploadx({ - directory: getResumableUploadPath(), - - expiration: { maxAge: undefined, rolling: true }, - - // Could be big with thumbnails/previews - maxMetadataSize: '10MB', - - logger: { - logLevel: CONFIG.LOG.LEVEL as LogLevel, - debug: logger.debug.bind(logger), - info: logger.info.bind(logger), - warn: logger.warn.bind(logger), - error: logger.error.bind(logger) - }, - - userIdentifier: (_, res: express.Response) => { - if (!res.locals.oauth) return undefined - - return res.locals.oauth.token.user.id + '' - }, - - filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}` -}) - -export { - uploadx -} diff --git a/server/lib/user.ts b/server/lib/user.ts deleted file mode 100644 index 56995cca3..000000000 --- a/server/lib/user.ts +++ /dev/null @@ -1,301 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { UserModel } from '@server/models/user/user' -import { MActorDefault } from '@server/types/models/actor' -import { ActivityPubActorType } from '../../shared/models/activitypub' -import { UserAdminFlag, UserNotificationSetting, UserNotificationSettingValue, UserRole } from '../../shared/models/users' -import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' -import { sequelizeTypescript } from '../initializers/database' -import { AccountModel } from '../models/account/account' -import { UserNotificationSettingModel } from '../models/user/user-notification-setting' -import { MAccountDefault, MChannelActor } from '../types/models' -import { MRegistration, MUser, MUserDefault, MUserId } from '../types/models/user' -import { generateAndSaveActorKeys } from './activitypub/actors' -import { getLocalAccountActivityPubUrl } from './activitypub/url' -import { Emailer } from './emailer' -import { LiveQuotaStore } from './live/live-quota-store' -import { buildActorInstance, findAvailableLocalActorName } from './local-actor' -import { Redis } from './redis' -import { createLocalVideoChannel } from './video-channel' -import { createWatchLaterPlaylist } from './video-playlist' - -type ChannelNames = { name: string, displayName: string } - -function buildUser (options: { - username: string - password: string - email: string - - role?: UserRole // Default to UserRole.User - adminFlags?: UserAdminFlag // Default to UserAdminFlag.NONE - - emailVerified: boolean | null - - videoQuota?: number // Default to CONFIG.USER.VIDEO_QUOTA - videoQuotaDaily?: number // Default to CONFIG.USER.VIDEO_QUOTA_DAILY - - pluginAuth?: string -}): MUser { - const { - username, - password, - email, - role = UserRole.USER, - emailVerified, - videoQuota = CONFIG.USER.VIDEO_QUOTA, - videoQuotaDaily = CONFIG.USER.VIDEO_QUOTA_DAILY, - adminFlags = UserAdminFlag.NONE, - pluginAuth - } = options - - return new UserModel({ - username, - password, - email, - - nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, - p2pEnabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED, - videosHistoryEnabled: CONFIG.USER.HISTORY.VIDEOS.ENABLED, - - autoPlayVideo: true, - - role, - emailVerified, - adminFlags, - - videoQuota, - videoQuotaDaily, - - pluginAuth - }) -} - -// --------------------------------------------------------------------------- - -async function createUserAccountAndChannelAndPlaylist (parameters: { - userToCreate: MUser - userDisplayName?: string - channelNames?: ChannelNames - validateUser?: boolean -}): Promise<{ user: MUserDefault, account: MAccountDefault, videoChannel: MChannelActor }> { - const { userToCreate, userDisplayName, channelNames, validateUser = true } = parameters - - const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { - const userOptions = { - transaction: t, - validate: validateUser - } - - const userCreated: MUserDefault = await userToCreate.save(userOptions) - userCreated.NotificationSetting = await createDefaultUserNotificationSettings(userCreated, t) - - const accountCreated = await createLocalAccountWithoutKeys({ - name: userCreated.username, - displayName: userDisplayName, - userId: userCreated.id, - applicationId: null, - t - }) - userCreated.Account = accountCreated - - const channelAttributes = await buildChannelAttributes({ user: userCreated, transaction: t, channelNames }) - const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t) - - const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) - - return { user: userCreated, account: accountCreated, videoChannel, videoPlaylist } - }) - - const [ accountActorWithKeys, channelActorWithKeys ] = await Promise.all([ - generateAndSaveActorKeys(account.Actor), - generateAndSaveActorKeys(videoChannel.Actor) - ]) - - account.Actor = accountActorWithKeys - videoChannel.Actor = channelActorWithKeys - - return { user, account, videoChannel } -} - -async function createLocalAccountWithoutKeys (parameters: { - name: string - displayName?: string - userId: number | null - applicationId: number | null - t: Transaction | undefined - type?: ActivityPubActorType -}) { - const { name, displayName, userId, applicationId, t, type = 'Person' } = parameters - const url = getLocalAccountActivityPubUrl(name) - - const actorInstance = buildActorInstance(type, url, name) - const actorInstanceCreated: MActorDefault = await actorInstance.save({ transaction: t }) - - const accountInstance = new AccountModel({ - name: displayName || name, - userId, - applicationId, - actorId: actorInstanceCreated.id - }) - - const accountInstanceCreated: MAccountDefault = await accountInstance.save({ transaction: t }) - accountInstanceCreated.Actor = actorInstanceCreated - - return accountInstanceCreated -} - -async function createApplicationActor (applicationId: number) { - const accountCreated = await createLocalAccountWithoutKeys({ - name: SERVER_ACTOR_NAME, - userId: null, - applicationId, - t: undefined, - type: 'Application' - }) - - accountCreated.Actor = await generateAndSaveActorKeys(accountCreated.Actor) - - return accountCreated -} - -// --------------------------------------------------------------------------- - -async function sendVerifyUserEmail (user: MUser, isPendingEmail = false) { - const verificationString = await Redis.Instance.setUserVerifyEmailVerificationString(user.id) - let verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?userId=${user.id}&verificationString=${verificationString}` - - if (isPendingEmail) verifyEmailUrl += '&isPendingEmail=true' - - const to = isPendingEmail - ? user.pendingEmail - : user.email - - const username = user.username - - Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: false }) -} - -async function sendVerifyRegistrationEmail (registration: MRegistration) { - const verificationString = await Redis.Instance.setRegistrationVerifyEmailVerificationString(registration.id) - const verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?registrationId=${registration.id}&verificationString=${verificationString}` - - const to = registration.email - const username = registration.username - - Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: true }) -} - -// --------------------------------------------------------------------------- - -async function getOriginalVideoFileTotalFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId', - daily: false - }) - - const base = await UserModel.getTotalRawQuery(query, user.id) - - return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) -} - -// Returns cumulative size of all video files uploaded in the last 24 hours. -async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId', - daily: true - }) - - const base = await UserModel.getTotalRawQuery(query, user.id) - - return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) -} - -async function isAbleToUploadVideo (userId: number, newVideoSize: number) { - const user = await UserModel.loadById(userId) - - if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) - - const [ totalBytes, totalBytesDaily ] = await Promise.all([ - getOriginalVideoFileTotalFromUser(user), - getOriginalVideoFileTotalDailyFromUser(user) - ]) - - const uploadedTotal = newVideoSize + totalBytes - const uploadedDaily = newVideoSize + totalBytesDaily - - logger.debug( - 'Check user %d quota to upload another video.', userId, - { totalBytes, totalBytesDaily, videoQuota: user.videoQuota, videoQuotaDaily: user.videoQuotaDaily, newVideoSize } - ) - - if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota - if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily - - return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily -} - -// --------------------------------------------------------------------------- - -export { - getOriginalVideoFileTotalFromUser, - getOriginalVideoFileTotalDailyFromUser, - createApplicationActor, - createUserAccountAndChannelAndPlaylist, - createLocalAccountWithoutKeys, - - sendVerifyUserEmail, - sendVerifyRegistrationEmail, - - isAbleToUploadVideo, - buildUser -} - -// --------------------------------------------------------------------------- - -function createDefaultUserNotificationSettings (user: MUserId, t: Transaction | undefined) { - const values: UserNotificationSetting & { userId: number } = { - userId: user.id, - newVideoFromSubscription: UserNotificationSettingValue.WEB, - newCommentOnMyVideo: UserNotificationSettingValue.WEB, - myVideoImportFinished: UserNotificationSettingValue.WEB, - myVideoPublished: UserNotificationSettingValue.WEB, - abuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newUserRegistration: UserNotificationSettingValue.WEB, - commentMention: UserNotificationSettingValue.WEB, - newFollow: UserNotificationSettingValue.WEB, - newInstanceFollower: UserNotificationSettingValue.WEB, - abuseNewMessage: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - abuseStateChange: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - autoInstanceFollowing: UserNotificationSettingValue.WEB, - newPeerTubeVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newPluginVersion: UserNotificationSettingValue.WEB, - myVideoStudioEditionFinished: UserNotificationSettingValue.WEB - } - - return UserNotificationSettingModel.create(values, { transaction: t }) -} - -async function buildChannelAttributes (options: { - user: MUser - transaction?: Transaction - channelNames?: ChannelNames -}) { - const { user, transaction, channelNames } = options - - if (channelNames) return channelNames - - const channelName = await findAvailableLocalActorName(user.username + '_channel', transaction) - const videoChannelDisplayName = `Main ${user.username} channel` - - return { - name: channelName, - displayName: videoChannelDisplayName - } -} diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts deleted file mode 100644 index d5664a1b9..000000000 --- a/server/lib/video-blacklist.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { Transaction } from 'sequelize' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { sequelizeTypescript } from '@server/initializers/database' -import { - MUser, - MVideoAccountLight, - MVideoBlacklist, - MVideoBlacklistVideo, - MVideoFullLight, - MVideoWithBlacklistLight -} from '@server/types/models' -import { LiveVideoError, UserRight, VideoBlacklistCreate, VideoBlacklistType } from '../../shared/models' -import { UserAdminFlag } from '../../shared/models/users/user-flag.model' -import { logger, loggerTagsFactory } from '../helpers/logger' -import { CONFIG } from '../initializers/config' -import { VideoBlacklistModel } from '../models/video/video-blacklist' -import { sendDeleteVideo } from './activitypub/send' -import { federateVideoIfNeeded } from './activitypub/videos' -import { LiveManager } from './live/live-manager' -import { Notifier } from './notifier' -import { Hooks } from './plugins/hooks' - -const lTags = loggerTagsFactory('blacklist') - -async function autoBlacklistVideoIfNeeded (parameters: { - video: MVideoWithBlacklistLight - user?: MUser - isRemote: boolean - isNew: boolean - isNewFile: boolean - notify?: boolean - transaction?: Transaction -}) { - const { video, user, isRemote, isNew, isNewFile, notify = true, transaction } = parameters - const doAutoBlacklist = await Hooks.wrapFun( - autoBlacklistNeeded, - { video, user, isRemote, isNew, isNewFile }, - 'filter:video.auto-blacklist.result' - ) - - if (!doAutoBlacklist) return false - - const videoBlacklistToCreate = { - videoId: video.id, - unfederated: true, - reason: 'Auto-blacklisted. Moderator review required.', - type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED - } - const [ videoBlacklist ] = await VideoBlacklistModel.findOrCreate({ - where: { - videoId: video.id - }, - defaults: videoBlacklistToCreate, - transaction - }) - video.VideoBlacklist = videoBlacklist - - videoBlacklist.Video = video - - if (notify) { - afterCommitIfTransaction(transaction, () => { - Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) - }) - } - - logger.info('Video %s auto-blacklisted.', video.uuid, lTags(video.uuid)) - - return true -} - -async function blacklistVideo (videoInstance: MVideoAccountLight, options: VideoBlacklistCreate) { - const blacklist: MVideoBlacklistVideo = await VideoBlacklistModel.create({ - videoId: videoInstance.id, - unfederated: options.unfederate === true, - reason: options.reason, - type: VideoBlacklistType.MANUAL - }) - blacklist.Video = videoInstance - - if (options.unfederate === true) { - await sendDeleteVideo(videoInstance, undefined) - } - - if (videoInstance.isLive) { - LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED) - } - - Notifier.Instance.notifyOnVideoBlacklist(blacklist) -} - -async function unblacklistVideo (videoBlacklist: MVideoBlacklist, video: MVideoFullLight) { - const videoBlacklistType = await sequelizeTypescript.transaction(async t => { - const unfederated = videoBlacklist.unfederated - const videoBlacklistType = videoBlacklist.type - - await videoBlacklist.destroy({ transaction: t }) - video.VideoBlacklist = undefined - - // Re federate the video - if (unfederated === true) { - await federateVideoIfNeeded(video, true, t) - } - - return videoBlacklistType - }) - - Notifier.Instance.notifyOnVideoUnblacklist(video) - - if (videoBlacklistType === VideoBlacklistType.AUTO_BEFORE_PUBLISHED) { - Notifier.Instance.notifyOnVideoPublishedAfterRemovedFromAutoBlacklist(video) - - // Delete on object so new video notifications will send - delete video.VideoBlacklist - Notifier.Instance.notifyOnNewVideoIfNeeded(video) - } -} - -// --------------------------------------------------------------------------- - -export { - autoBlacklistVideoIfNeeded, - blacklistVideo, - unblacklistVideo -} - -// --------------------------------------------------------------------------- - -function autoBlacklistNeeded (parameters: { - video: MVideoWithBlacklistLight - isRemote: boolean - isNew: boolean - isNewFile: boolean - user?: MUser -}) { - const { user, video, isRemote, isNew, isNewFile } = parameters - - // Already blacklisted - if (video.VideoBlacklist) return false - if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED || !user) return false - if (isRemote || (isNew === false && isNewFile === false)) return false - - if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST)) return false - - return true -} diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts deleted file mode 100644 index 8322c9ad2..000000000 --- a/server/lib/video-channel.ts +++ /dev/null @@ -1,50 +0,0 @@ -import * as Sequelize from 'sequelize' -import { VideoChannelCreate } from '../../shared/models' -import { VideoModel } from '../models/video/video' -import { VideoChannelModel } from '../models/video/video-channel' -import { MAccountId, MChannelId } from '../types/models' -import { getLocalVideoChannelActivityPubUrl } from './activitypub/url' -import { federateVideoIfNeeded } from './activitypub/videos' -import { buildActorInstance } from './local-actor' - -async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { - const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) - const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name) - - const actorInstanceCreated = await actorInstance.save({ transaction: t }) - - const videoChannelData = { - name: videoChannelInfo.displayName, - description: videoChannelInfo.description, - support: videoChannelInfo.support, - accountId: account.id, - actorId: actorInstanceCreated.id - } - - const videoChannel = new VideoChannelModel(videoChannelData) - - const options = { transaction: t } - const videoChannelCreated = await videoChannel.save(options) - - videoChannelCreated.Actor = actorInstanceCreated - - // No need to send this empty video channel to followers - return videoChannelCreated -} - -async function federateAllVideosOfChannel (videoChannel: MChannelId) { - const videoIds = await VideoModel.getAllIdsFromChannel(videoChannel) - - for (const videoId of videoIds) { - const video = await VideoModel.loadFull(videoId) - - await federateVideoIfNeeded(video, false) - } -} - -// --------------------------------------------------------------------------- - -export { - createLocalVideoChannel, - federateAllVideosOfChannel -} diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts deleted file mode 100644 index 6eb865f7f..000000000 --- a/server/lib/video-comment.ts +++ /dev/null @@ -1,116 +0,0 @@ -import express from 'express' -import { cloneDeep } from 'lodash' -import * as Sequelize from 'sequelize' -import { logger } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' -import { ResultList } from '../../shared/models' -import { VideoCommentThreadTree } from '../../shared/models/videos/comment/video-comment.model' -import { VideoCommentModel } from '../models/video/video-comment' -import { - MAccountDefault, - MComment, - MCommentFormattable, - MCommentOwnerVideo, - MCommentOwnerVideoReply, - MVideoFullLight -} from '../types/models' -import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send' -import { getLocalVideoCommentActivityPubUrl } from './activitypub/url' -import { Hooks } from './plugins/hooks' - -async function removeComment (commentArg: MComment, req: express.Request, res: express.Response) { - let videoCommentInstanceBefore: MCommentOwnerVideo - - await sequelizeTypescript.transaction(async t => { - const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(commentArg.url, t) - - videoCommentInstanceBefore = cloneDeep(comment) - - if (comment.isOwned() || comment.Video.isOwned()) { - await sendDeleteVideoComment(comment, t) - } - - comment.markAsDeleted() - - await comment.save({ transaction: t }) - - logger.info('Video comment %d deleted.', comment.id) - }) - - Hooks.runAction('action:api.video-comment.deleted', { comment: videoCommentInstanceBefore, req, res }) -} - -async function createVideoComment (obj: { - text: string - inReplyToComment: MComment | null - video: MVideoFullLight - account: MAccountDefault -}, t: Sequelize.Transaction) { - let originCommentId: number | null = null - let inReplyToCommentId: number | null = null - - if (obj.inReplyToComment && obj.inReplyToComment !== null) { - originCommentId = obj.inReplyToComment.originCommentId || obj.inReplyToComment.id - inReplyToCommentId = obj.inReplyToComment.id - } - - const comment = await VideoCommentModel.create({ - text: obj.text, - originCommentId, - inReplyToCommentId, - videoId: obj.video.id, - accountId: obj.account.id, - url: new Date().toISOString() - }, { transaction: t, validate: false }) - - comment.url = getLocalVideoCommentActivityPubUrl(obj.video, comment) - - const savedComment: MCommentOwnerVideoReply = await comment.save({ transaction: t }) - savedComment.InReplyToVideoComment = obj.inReplyToComment - savedComment.Video = obj.video - savedComment.Account = obj.account - - await sendCreateVideoComment(savedComment, t) - - return savedComment -} - -function buildFormattedCommentTree (resultList: ResultList): VideoCommentThreadTree { - // Comments are sorted by id ASC - const comments = resultList.data - - const comment = comments.shift() - const thread: VideoCommentThreadTree = { - comment: comment.toFormattedJSON(), - children: [] - } - const idx = { - [comment.id]: thread - } - - while (comments.length !== 0) { - const childComment = comments.shift() - - const childCommentThread: VideoCommentThreadTree = { - comment: childComment.toFormattedJSON(), - children: [] - } - - const parentCommentThread = idx[childComment.inReplyToCommentId] - // Maybe the parent comment was blocked by the admin/user - if (!parentCommentThread) continue - - parentCommentThread.children.push(childCommentThread) - idx[childComment.id] = childCommentThread - } - - return thread -} - -// --------------------------------------------------------------------------- - -export { - removeComment, - createVideoComment, - buildFormattedCommentTree -} diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts deleted file mode 100644 index 46af67ccd..000000000 --- a/server/lib/video-file.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { logger } from '@server/helpers/logger' -import { VideoFileModel } from '@server/models/video/video-file' -import { MVideoWithAllFiles } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { getFileSize } from '@shared/extra-utils' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' -import { VideoFileMetadata, VideoResolution } from '@shared/models' -import { lTags } from './object-storage/shared' -import { generateHLSVideoFilename, generateWebVideoFilename } from './paths' -import { VideoPathManager } from './video-path-manager' - -async function buildNewFile (options: { - path: string - mode: 'web-video' | 'hls' -}) { - const { path, mode } = options - - const probe = await ffprobePromise(path) - const size = await getFileSize(path) - - const videoFile = new VideoFileModel({ - extname: getLowercaseExtension(path), - size, - metadata: await buildFileMetadata(path, probe) - }) - - if (await isAudioFile(path, probe)) { - videoFile.resolution = VideoResolution.H_NOVIDEO - } else { - videoFile.fps = await getVideoStreamFPS(path, probe) - videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution - } - - videoFile.filename = mode === 'web-video' - ? generateWebVideoFilename(videoFile.resolution, videoFile.extname) - : generateHLSVideoFilename(videoFile.resolution) - - return videoFile -} - -// --------------------------------------------------------------------------- - -async function removeHLSPlaylist (video: MVideoWithAllFiles) { - const hls = video.getHLSPlaylist() - if (!hls) return - - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.removeStreamingPlaylistFiles(hls) - await hls.destroy() - - video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id) - } finally { - videoFileMutexReleaser() - } -} - -async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { - logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid)) - - const hls = video.getHLSPlaylist() - const files = hls.VideoFiles - - if (files.length === 1) { - await removeHLSPlaylist(video) - return undefined - } - - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - const toDelete = files.find(f => f.id === fileToDeleteId) - await video.removeStreamingPlaylistVideoFile(video.getHLSPlaylist(), toDelete) - await toDelete.destroy() - - hls.VideoFiles = hls.VideoFiles.filter(f => f.id !== toDelete.id) - } finally { - videoFileMutexReleaser() - } - - return hls -} - -// --------------------------------------------------------------------------- - -async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - for (const file of video.VideoFiles) { - await video.removeWebVideoFile(file) - await file.destroy() - } - - video.VideoFiles = [] - } finally { - videoFileMutexReleaser() - } - - return video -} - -async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) { - const files = video.VideoFiles - - if (files.length === 1) { - return removeAllWebVideoFiles(video) - } - - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - try { - const toDelete = files.find(f => f.id === fileToDeleteId) - await video.removeWebVideoFile(toDelete) - await toDelete.destroy() - - video.VideoFiles = files.filter(f => f.id !== toDelete.id) - } finally { - videoFileMutexReleaser() - } - - return video -} - -// --------------------------------------------------------------------------- - -async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { - const metadata = existingProbe || await ffprobePromise(path) - - return new VideoFileMetadata(metadata) -} - -// --------------------------------------------------------------------------- - -export { - buildNewFile, - - removeHLSPlaylist, - removeHLSFile, - removeAllWebVideoFiles, - removeWebVideoFile, - - buildFileMetadata -} diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts deleted file mode 100644 index 133544bb2..000000000 --- a/server/lib/video-path-manager.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { Mutex } from 'async-mutex' -import { remove } from 'fs-extra' -import { extname, join } from 'path' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { extractVideo } from '@server/helpers/video' -import { CONFIG } from '@server/initializers/config' -import { DIRECTORIES } from '@server/initializers/constants' -import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models' -import { buildUUID } from '@shared/extra-utils' -import { VideoStorage } from '@shared/models' -import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage' -import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' -import { isVideoInPrivateDirectory } from './video-privacy' - -type MakeAvailableCB = (path: string) => Promise | T - -const lTags = loggerTagsFactory('video-path-manager') - -class VideoPathManager { - - private static instance: VideoPathManager - - // Key is a video UUID - private readonly videoFileMutexStore = new Map() - - private constructor () {} - - getFSHLSOutputPath (video: MVideo, filename?: string) { - const base = getHLSDirectory(video) - if (!filename) return base - - return join(base, filename) - } - - getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - if (videoFile.isHLS()) { - const video = extractVideo(videoOrPlaylist) - - return join(getHLSRedundancyDirectory(video), videoFile.filename) - } - - return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) - } - - getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - const video = extractVideo(videoOrPlaylist) - - if (videoFile.isHLS()) { - return join(getHLSDirectory(video), videoFile.filename) - } - - if (isVideoInPrivateDirectory(video.privacy)) { - return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename) - } - - return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename) - } - - async makeAvailableVideoFile (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { - if (videoFile.storage === VideoStorage.FILE_SYSTEM) { - return this.makeAvailableFactory( - () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile), - false, - cb - ) - } - - const destination = this.buildTMPDestination(videoFile.filename) - - if (videoFile.isHLS()) { - const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist - - return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, videoFile.filename, destination), - true, - cb - ) - } - - return this.makeAvailableFactory( - () => makeWebVideoFileAvailable(videoFile.filename, destination), - true, - cb - ) - } - - async makeAvailableResolutionPlaylistFile (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { - const filename = getHlsResolutionPlaylistFilename(videoFile.filename) - - if (videoFile.storage === VideoStorage.FILE_SYSTEM) { - return this.makeAvailableFactory( - () => join(getHLSDirectory(videoFile.getVideo()), filename), - false, - cb - ) - } - - const playlist = videoFile.VideoStreamingPlaylist - return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), - true, - cb - ) - } - - async makeAvailablePlaylistFile (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB) { - if (playlist.storage === VideoStorage.FILE_SYSTEM) { - return this.makeAvailableFactory( - () => join(getHLSDirectory(playlist.Video), filename), - false, - cb - ) - } - - return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), - true, - cb - ) - } - - async lockFiles (videoUUID: string) { - if (!this.videoFileMutexStore.has(videoUUID)) { - this.videoFileMutexStore.set(videoUUID, new Mutex()) - } - - const mutex = this.videoFileMutexStore.get(videoUUID) - const releaser = await mutex.acquire() - - logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID)) - - return releaser - } - - unlockFiles (videoUUID: string) { - const mutex = this.videoFileMutexStore.get(videoUUID) - - mutex.release() - - logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID)) - } - - private async makeAvailableFactory (method: () => Promise | string, clean: boolean, cb: MakeAvailableCB) { - let result: T - - const destination = await method() - - try { - result = await cb(destination) - } catch (err) { - if (destination && clean) await remove(destination) - throw err - } - - if (clean) await remove(destination) - - return result - } - - private buildTMPDestination (filename: string) { - return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename)) - - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - VideoPathManager -} diff --git a/server/lib/video-playlist.ts b/server/lib/video-playlist.ts deleted file mode 100644 index a1af2e1af..000000000 --- a/server/lib/video-playlist.ts +++ /dev/null @@ -1,30 +0,0 @@ -import * as Sequelize from 'sequelize' -import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model' -import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model' -import { VideoPlaylistModel } from '../models/video/video-playlist' -import { MAccount } from '../types/models' -import { MVideoPlaylistOwner } from '../types/models/video/video-playlist' -import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url' - -async function createWatchLaterPlaylist (account: MAccount, t: Sequelize.Transaction) { - const videoPlaylist: MVideoPlaylistOwner = new VideoPlaylistModel({ - name: 'Watch later', - privacy: VideoPlaylistPrivacy.PRIVATE, - type: VideoPlaylistType.WATCH_LATER, - ownerAccountId: account.id - }) - - videoPlaylist.url = getLocalVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object - - await videoPlaylist.save({ transaction: t }) - - videoPlaylist.OwnerAccount = account - - return videoPlaylist -} - -// --------------------------------------------------------------------------- - -export { - createWatchLaterPlaylist -} diff --git a/server/lib/video-pre-import.ts b/server/lib/video-pre-import.ts deleted file mode 100644 index fcb9f77d7..000000000 --- a/server/lib/video-pre-import.ts +++ /dev/null @@ -1,323 +0,0 @@ -import { remove } from 'fs-extra' -import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils' -import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' -import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' -import { isResolvingToUnicastOnly } from '@server/helpers/dns' -import { logger } from '@server/helpers/logger' -import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl' -import { CONFIG } from '@server/initializers/config' -import { sequelizeTypescript } from '@server/initializers/database' -import { Hooks } from '@server/lib/plugins/hooks' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { setVideoTags } from '@server/lib/video' -import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' -import { VideoModel } from '@server/models/video/video' -import { VideoCaptionModel } from '@server/models/video/video-caption' -import { VideoImportModel } from '@server/models/video/video-import' -import { FilteredModelAttributes } from '@server/types' -import { - MChannelAccountDefault, - MChannelSync, - MThumbnail, - MUser, - MVideoAccountDefault, - MVideoCaption, - MVideoImportFormattable, - MVideoTag, - MVideoThumbnail, - MVideoWithBlacklistLight -} from '@server/types/models' -import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' -import { getLocalVideoActivityPubUrl } from './activitypub/url' -import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail' -import { VideoPasswordModel } from '@server/models/video/video-password' - -class YoutubeDlImportError extends Error { - code: YoutubeDlImportError.CODE - cause?: Error // Property to remove once ES2022 is used - constructor ({ message, code }) { - super(message) - this.code = code - } - - static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) { - const ytDlErr = new this({ message: message ?? err.message, code }) - ytDlErr.cause = err - ytDlErr.stack = err.stack // Useless once ES2022 is used - return ytDlErr - } -} - -namespace YoutubeDlImportError { - export enum CODE { - FETCH_ERROR, - NOT_ONLY_UNICAST_URL - } -} - -// --------------------------------------------------------------------------- - -async function insertFromImportIntoDB (parameters: { - video: MVideoThumbnail - thumbnailModel: MThumbnail - previewModel: MThumbnail - videoChannel: MChannelAccountDefault - tags: string[] - videoImportAttributes: FilteredModelAttributes - user: MUser - videoPasswords?: string[] -}): Promise { - const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user, videoPasswords } = parameters - - const videoImport = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - // Save video object in database - const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) - videoCreated.VideoChannel = videoChannel - - if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) - if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) - - if (videoCreated.privacy === VideoPrivacy.PASSWORD_PROTECTED) { - await VideoPasswordModel.addPasswords(videoPasswords, video.id, t) - } - - await autoBlacklistVideoIfNeeded({ - video: videoCreated, - user, - notify: false, - isRemote: false, - isNew: true, - isNewFile: true, - transaction: t - }) - - await setVideoTags({ video: videoCreated, tags, transaction: t }) - - // Create video import object in database - const videoImport = await VideoImportModel.create( - Object.assign({ videoId: videoCreated.id }, videoImportAttributes), - sequelizeOptions - ) as MVideoImportFormattable - videoImport.Video = videoCreated - - return videoImport - }) - - return videoImport -} - -async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: { - channelId: number - importData: YoutubeDLInfo - importDataOverride?: Partial - importType: 'url' | 'torrent' -}): Promise { - let videoData = { - name: importDataOverride?.name || importData.name || 'Unknown name', - remote: false, - category: importDataOverride?.category || importData.category, - licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, - language: importDataOverride?.language || importData.language, - commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, - downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, - waitTranscoding: importDataOverride?.waitTranscoding ?? true, - state: VideoState.TO_IMPORT, - nsfw: importDataOverride?.nsfw || importData.nsfw || false, - description: importDataOverride?.description || importData.description, - support: importDataOverride?.support || null, - privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE, - duration: 0, // duration will be set by the import job - channelId, - originallyPublishedAt: importDataOverride?.originallyPublishedAt - ? new Date(importDataOverride?.originallyPublishedAt) - : importData.originallyPublishedAtWithoutTime - } - - videoData = await Hooks.wrapObject( - videoData, - importType === 'url' - ? 'filter:api.video.import-url.video-attribute.result' - : 'filter:api.video.import-torrent.video-attribute.result' - ) - - const video = new VideoModel(videoData) - video.url = getLocalVideoActivityPubUrl(video) - - return video -} - -async function buildYoutubeDLImport (options: { - targetUrl: string - channel: MChannelAccountDefault - user: MUser - channelSync?: MChannelSync - importDataOverride?: Partial - thumbnailFilePath?: string - previewFilePath?: string -}) { - const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options - - const youtubeDL = new YoutubeDLWrapper( - targetUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - // Get video infos - let youtubeDLInfo: YoutubeDLInfo - try { - youtubeDLInfo = await youtubeDL.getInfoForDownload() - } catch (err) { - throw YoutubeDlImportError.fromError( - err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}` - ) - } - - if (!await hasUnicastURLsOnly(youtubeDLInfo)) { - throw new YoutubeDlImportError({ - message: 'Cannot use non unicast IP as targetUrl.', - code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL - }) - } - - const video = await buildVideoFromImport({ - channelId: channel.id, - importData: youtubeDLInfo, - importDataOverride, - importType: 'url' - }) - - const thumbnailModel = await forgeThumbnail({ - inputPath: thumbnailFilePath, - downloadUrl: youtubeDLInfo.thumbnailUrl, - video, - type: ThumbnailType.MINIATURE - }) - - const previewModel = await forgeThumbnail({ - inputPath: previewFilePath, - downloadUrl: youtubeDLInfo.thumbnailUrl, - video, - type: ThumbnailType.PREVIEW - }) - - const videoImport = await insertFromImportIntoDB({ - video, - thumbnailModel, - previewModel, - videoChannel: channel, - tags: importDataOverride?.tags || youtubeDLInfo.tags, - user, - videoImportAttributes: { - targetUrl, - state: VideoImportState.PENDING, - userId: user.id, - videoChannelSyncId: channelSync?.id - }, - videoPasswords: importDataOverride.videoPasswords - }) - - // Get video subtitles - await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) - - let fileExt = `.${youtubeDLInfo.ext}` - if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' - - const payload: VideoImportPayload = { - type: 'youtube-dl' as 'youtube-dl', - videoImportId: videoImport.id, - fileExt, - // If part of a sync process, there is a parent job that will aggregate children results - preventException: !!channelSync - } - - return { - videoImport, - job: { type: 'video-import' as 'video-import', payload } - } -} - -// --------------------------------------------------------------------------- - -export { - buildYoutubeDLImport, - YoutubeDlImportError, - insertFromImportIntoDB, - buildVideoFromImport -} - -// --------------------------------------------------------------------------- - -async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { - inputPath?: string - downloadUrl?: string - video: MVideoThumbnail - type: ThumbnailType -}): Promise { - if (inputPath) { - return updateLocalVideoMiniatureFromExisting({ - inputPath, - video, - type, - automaticallyGenerated: false - }) - } - - if (downloadUrl) { - try { - return await updateLocalVideoMiniatureFromUrl({ downloadUrl, video, type }) - } catch (err) { - logger.warn('Cannot process thumbnail %s from youtube-dl.', downloadUrl, { err }) - } - } - - return null -} - -async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { - try { - const subtitles = await youtubeDL.getSubtitles() - - logger.info('Found %s subtitles candidates from youtube-dl import %s.', subtitles.length, targetUrl) - - for (const subtitle of subtitles) { - if (!await isVTTFileValid(subtitle.path)) { - logger.info('%s is not a valid youtube-dl subtitle, skipping', subtitle.path) - await remove(subtitle.path) - continue - } - - const videoCaption = new VideoCaptionModel({ - videoId, - language: subtitle.language, - filename: VideoCaptionModel.generateCaptionName(subtitle.language) - }) as MVideoCaption - - // Move physical file - await moveAndProcessCaptionFile(subtitle, videoCaption) - - await sequelizeTypescript.transaction(async t => { - await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) - }) - - logger.info('Added %s youtube-dl subtitle', subtitle.path) - } - } catch (err) { - logger.warn('Cannot get video subtitles.', { err }) - } -} - -async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { - const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) - const uniqHosts = new Set(hosts) - - for (const h of uniqHosts) { - if (await isResolvingToUnicastOnly(h) !== true) { - return false - } - } - - return true -} diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts deleted file mode 100644 index 5dd4d9781..000000000 --- a/server/lib/video-privacy.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { move } from 'fs-extra' -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { DIRECTORIES } from '@server/initializers/constants' -import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' -import { VideoPrivacy, VideoStorage } from '@shared/models' -import { updateHLSFilesACL, updateWebVideoFileACL } from './object-storage' - -const validPrivacySet = new Set([ - VideoPrivacy.PRIVATE, - VideoPrivacy.INTERNAL, - VideoPrivacy.PASSWORD_PROTECTED -]) - -function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { - if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { - video.publishedAt = new Date() - } - - video.privacy = newPrivacy -} - -function isVideoInPrivateDirectory (privacy) { - return validPrivacySet.has(privacy) -} - -function isVideoInPublicDirectory (privacy: VideoPrivacy) { - return !isVideoInPrivateDirectory(privacy) -} - -async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) { - // Now public, previously private - if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) { - await moveFiles({ type: 'private-to-public', video }) - - return true - } - - // Now private, previously public - if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) { - await moveFiles({ type: 'public-to-private', video }) - - return true - } - - return false -} - -export { - setVideoPrivacy, - - isVideoInPrivateDirectory, - isVideoInPublicDirectory, - - moveFilesIfPrivacyChanged -} - -// --------------------------------------------------------------------------- - -type MoveType = 'private-to-public' | 'public-to-private' - -async function moveFiles (options: { - type: MoveType - video: MVideoFullLight -}) { - const { type, video } = options - - for (const file of video.VideoFiles) { - if (file.storage === VideoStorage.FILE_SYSTEM) { - await moveWebVideoFileOnFS(type, video, file) - } else { - await updateWebVideoFileACL(video, file) - } - } - - const hls = video.getHLSPlaylist() - - if (hls) { - if (hls.storage === VideoStorage.FILE_SYSTEM) { - await moveHLSFilesOnFS(type, video) - } else { - await updateHLSFilesACL(hls) - } - } -} - -async function moveWebVideoFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { - const directories = getWebVideoDirectories(type) - - const source = join(directories.old, file.filename) - const destination = join(directories.new, file.filename) - - try { - logger.info('Moving web video files of %s after privacy change (%s -> %s).', video.uuid, source, destination) - - await move(source, destination) - } catch (err) { - logger.error('Cannot move web video file %s to %s after privacy change', source, destination, { err }) - } -} - -function getWebVideoDirectories (moveType: MoveType) { - if (moveType === 'private-to-public') { - return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } - } - - return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } -} - -// --------------------------------------------------------------------------- - -async function moveHLSFilesOnFS (type: MoveType, video: MVideo) { - const directories = getHLSDirectories(type) - - const source = join(directories.old, video.uuid) - const destination = join(directories.new, video.uuid) - - try { - logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) - - await move(source, destination) - } catch (err) { - logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) - } -} - -function getHLSDirectories (moveType: MoveType) { - if (moveType === 'private-to-public') { - return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } - } - - return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } -} diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts deleted file mode 100644 index 893725d85..000000000 --- a/server/lib/video-state.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { Transaction } from 'sequelize' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { sequelizeTypescript } from '@server/initializers/database' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models' -import { VideoState } from '@shared/models' -import { federateVideoIfNeeded } from './activitypub/videos' -import { JobQueue } from './job-queue' -import { Notifier } from './notifier' -import { buildMoveToObjectStorageJob } from './video' - -function buildNextVideoState (currentState?: VideoState) { - if (currentState === VideoState.PUBLISHED) { - throw new Error('Video is already in its final state') - } - - if ( - currentState !== VideoState.TO_EDIT && - currentState !== VideoState.TO_TRANSCODE && - currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && - CONFIG.TRANSCODING.ENABLED - ) { - return VideoState.TO_TRANSCODE - } - - if ( - currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && - CONFIG.OBJECT_STORAGE.ENABLED - ) { - return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE - } - - return VideoState.PUBLISHED -} - -function moveToNextState (options: { - video: MVideoUUID - previousVideoState?: VideoState - isNewVideo?: boolean // Default true -}) { - const { video, previousVideoState, isNewVideo = true } = options - - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - // Maybe the video changed in database, refresh it - const videoDatabase = await VideoModel.loadFull(video.uuid, t) - // Video does not exist anymore - if (!videoDatabase) return undefined - - // Already in its final state - if (videoDatabase.state === VideoState.PUBLISHED) { - return federateVideoIfNeeded(videoDatabase, false, t) - } - - const newState = buildNextVideoState(videoDatabase.state) - - if (newState === VideoState.PUBLISHED) { - return moveToPublishedState({ video: videoDatabase, previousVideoState, isNewVideo, transaction: t }) - } - - if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - return moveToExternalStorageState({ video: videoDatabase, isNewVideo, transaction: t }) - } - }) - }) -} - -async function moveToExternalStorageState (options: { - video: MVideoFullLight - isNewVideo: boolean - transaction: Transaction -}) { - const { video, isNewVideo, transaction } = options - - const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) - const pendingTranscode = videoJobInfo?.pendingTranscode || 0 - - // We want to wait all transcoding jobs before moving the video on an external storage - if (pendingTranscode !== 0) return false - - const previousVideoState = video.state - - if (video.state !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction) - } - - logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) - - try { - await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState, isNewVideo })) - - return true - } catch (err) { - logger.error('Cannot add move to object storage job', { err }) - - return false - } -} - -function moveToFailedTranscodingState (video: MVideo) { - if (video.state === VideoState.TRANSCODING_FAILED) return - - return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) -} - -function moveToFailedMoveToObjectStorageState (video: MVideo) { - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return - - return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined) -} - -// --------------------------------------------------------------------------- - -export { - buildNextVideoState, - moveToExternalStorageState, - moveToFailedTranscodingState, - moveToFailedMoveToObjectStorageState, - moveToNextState -} - -// --------------------------------------------------------------------------- - -async function moveToPublishedState (options: { - video: MVideoFullLight - isNewVideo: boolean - transaction: Transaction - previousVideoState?: VideoState -}) { - const { video, isNewVideo, transaction, previousVideoState } = options - const previousState = previousVideoState ?? video.state - - logger.info('Publishing video %s.', video.uuid, { isNewVideo, previousState, tags: [ video.uuid ] }) - - await video.setNewState(VideoState.PUBLISHED, isNewVideo, transaction) - - await federateVideoIfNeeded(video, isNewVideo, transaction) - - if (previousState === VideoState.TO_EDIT) { - Notifier.Instance.notifyOfFinishedVideoStudioEdition(video) - return - } - - if (isNewVideo) { - Notifier.Instance.notifyOnNewVideoIfNeeded(video) - - if (previousState === VideoState.TO_TRANSCODE) { - Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) - } - } -} diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts deleted file mode 100644 index f549a7084..000000000 --- a/server/lib/video-studio.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { move, remove } from 'fs-extra' -import { join } from 'path' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' -import { CONFIG } from '@server/initializers/config' -import { UserModel } from '@server/models/user/user' -import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models' -import { getVideoStreamDuration } from '@shared/ffmpeg' -import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@shared/models' -import { federateVideoIfNeeded } from './activitypub/videos' -import { JobQueue } from './job-queue' -import { VideoStudioTranscodingJobHandler } from './runners' -import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job' -import { getTranscodingJobPriority } from './transcoding/transcoding-priority' -import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file' -import { VideoPathManager } from './video-path-manager' - -const lTags = loggerTagsFactory('video-studio') - -export function buildTaskFileFieldname (indice: number, fieldName = 'file') { - return `tasks[${indice}][options][${fieldName}]` -} - -export function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { - return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) -} - -export function getStudioTaskFilePath (filename: string) { - return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, filename) -} - -export async function safeCleanupStudioTMPFiles (tasks: VideoStudioTaskPayload[]) { - logger.info('Removing studio task files', { tasks, ...lTags() }) - - for (const task of tasks) { - try { - if (task.name === 'add-intro' || task.name === 'add-outro') { - await remove(task.options.file) - } else if (task.name === 'add-watermark') { - await remove(task.options.file) - } - } catch (err) { - logger.error('Cannot remove studio file', { err }) - } - } -} - -// --------------------------------------------------------------------------- - -export async function approximateIntroOutroAdditionalSize ( - video: MVideoFullLight, - tasks: VideoStudioTask[], - fileFinder: (i: number) => string -) { - let additionalDuration = 0 - - for (let i = 0; i < tasks.length; i++) { - const task = tasks[i] - - if (task.name !== 'add-intro' && task.name !== 'add-outro') continue - - const filePath = fileFinder(i) - additionalDuration += await getVideoStreamDuration(filePath) - } - - return (video.getMaxQualityFile().size / video.duration) * additionalDuration -} - -// --------------------------------------------------------------------------- - -export async function createVideoStudioJob (options: { - video: MVideo - user: MUser - payload: VideoStudioEditionPayload -}) { - const { video, user, payload } = options - - const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 }) - - if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) { - await new VideoStudioTranscodingJobHandler().create({ video, tasks: payload.tasks, priority }) - return - } - - await JobQueue.Instance.createJob({ type: 'video-studio-edition', payload, priority }) -} - -export async function onVideoStudioEnded (options: { - editionResultPath: string - tasks: VideoStudioTaskPayload[] - video: MVideoFullLight -}) { - const { video, tasks, editionResultPath } = options - - const newFile = await buildNewFile({ path: editionResultPath, mode: 'web-video' }) - newFile.videoId = video.id - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) - await move(editionResultPath, outputPath) - - await safeCleanupStudioTMPFiles(tasks) - - await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) - await removeAllFiles(video, newFile) - - await newFile.save() - - video.duration = await getVideoStreamDuration(outputPath) - await video.save() - - await federateVideoIfNeeded(video, false, undefined) - - const user = await UserModel.loadByVideoId(video.id) - - await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -async function removeAllFiles (video: MVideoWithAllFiles, webVideoFileException: MVideoFile) { - await removeHLSPlaylist(video) - - for (const file of video.VideoFiles) { - if (file.id === webVideoFileException.id) continue - - await removeWebVideoFile(video, file.id) - } -} diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts deleted file mode 100644 index e28e55cf7..000000000 --- a/server/lib/video-tokens-manager.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { LRUCache } from 'lru-cache' -import { LRU_CACHE } from '@server/initializers/constants' -import { MUserAccountUrl } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' - -// --------------------------------------------------------------------------- -// Create temporary tokens that can be used as URL query parameters to access video static files -// --------------------------------------------------------------------------- - -class VideoTokensManager { - - private static instance: VideoTokensManager - - private readonly lruCache = new LRUCache({ - max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE, - ttl: LRU_CACHE.VIDEO_TOKENS.TTL - }) - - private constructor () {} - - createForAuthUser (options: { - user: MUserAccountUrl - videoUUID: string - }) { - const { token, expires } = this.generateVideoToken() - - this.lruCache.set(token, pick(options, [ 'user', 'videoUUID' ])) - - return { token, expires } - } - - createForPasswordProtectedVideo (options: { - videoUUID: string - }) { - const { token, expires } = this.generateVideoToken() - - this.lruCache.set(token, pick(options, [ 'videoUUID' ])) - - return { token, expires } - } - - hasToken (options: { - token: string - videoUUID: string - }) { - const value = this.lruCache.get(options.token) - if (!value) return false - - return value.videoUUID === options.videoUUID - } - - getUserFromToken (options: { - token: string - }) { - const value = this.lruCache.get(options.token) - if (!value) return undefined - - return value.user - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - private generateVideoToken () { - const token = buildUUID() - const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL) - - return { token, expires } - } -} - -// --------------------------------------------------------------------------- - -export { - VideoTokensManager -} diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts deleted file mode 100644 index 0597488ad..000000000 --- a/server/lib/video-urls.ts +++ /dev/null @@ -1,31 +0,0 @@ - -import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' -import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' - -// ################## Redundancy ################## - -function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { - // Base URL used by our HLS player - return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid -} - -function generateWebVideoRedundancyUrl (file: MVideoFile) { - return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename -} - -// ################## Meta data ################## - -function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { - const path = '/api/v1/videos/' - - return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id -} - -// --------------------------------------------------------------------------- - -export { - getLocalVideoFileMetadataUrl, - - generateWebVideoRedundancyUrl, - generateHLSRedundancyUrl -} diff --git a/server/lib/video.ts b/server/lib/video.ts deleted file mode 100644 index 362c861a5..000000000 --- a/server/lib/video.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { UploadFiles } from 'express' -import memoizee from 'memoizee' -import { Transaction } from 'sequelize/types' -import { CONFIG } from '@server/initializers/config' -import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' -import { TagModel } from '@server/models/video/tag' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { FilteredModelAttributes } from '@server/types' -import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' -import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' -import { CreateJobArgument, JobQueue } from './job-queue/job-queue' -import { updateLocalVideoMiniatureFromExisting } from './thumbnail' -import { moveFilesIfPrivacyChanged } from './video-privacy' - -function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes { - return { - name: videoInfo.name, - remote: false, - category: videoInfo.category, - licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, - language: videoInfo.language, - commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, - downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, - waitTranscoding: videoInfo.waitTranscoding || false, - nsfw: videoInfo.nsfw || false, - description: videoInfo.description, - support: videoInfo.support, - privacy: videoInfo.privacy || VideoPrivacy.PRIVATE, - channelId, - originallyPublishedAt: videoInfo.originallyPublishedAt - ? new Date(videoInfo.originallyPublishedAt) - : null - } -} - -async function buildVideoThumbnailsFromReq (options: { - video: MVideoThumbnail - files: UploadFiles - fallback: (type: ThumbnailType) => Promise - automaticallyGenerated?: boolean -}) { - const { video, files, fallback, automaticallyGenerated } = options - - const promises = [ - { - type: ThumbnailType.MINIATURE, - fieldName: 'thumbnailfile' - }, - { - type: ThumbnailType.PREVIEW, - fieldName: 'previewfile' - } - ].map(p => { - const fields = files?.[p.fieldName] - - if (fields) { - return updateLocalVideoMiniatureFromExisting({ - inputPath: fields[0].path, - video, - type: p.type, - automaticallyGenerated: automaticallyGenerated || false - }) - } - - return fallback(p.type) - }) - - return Promise.all(promises) -} - -// --------------------------------------------------------------------------- - -async function setVideoTags (options: { - video: MVideoTag - tags: string[] - transaction?: Transaction -}) { - const { video, tags, transaction } = options - - const internalTags = tags || [] - const tagInstances = await TagModel.findOrCreateTags(internalTags, transaction) - - await video.$set('Tags', tagInstances, { transaction }) - video.Tags = tagInstances -} - -// --------------------------------------------------------------------------- - -async function buildMoveToObjectStorageJob (options: { - video: MVideoUUID - previousVideoState: VideoState - isNewVideo?: boolean // Default true -}) { - const { video, previousVideoState, isNewVideo = true } = options - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') - - return { - type: 'move-to-object-storage' as 'move-to-object-storage', - payload: { - videoUUID: video.uuid, - isNewVideo, - previousVideoState - } - } -} - -// --------------------------------------------------------------------------- - -async function getVideoDuration (videoId: number | string) { - const video = await VideoModel.load(videoId) - - const duration = video.isLive - ? undefined - : video.duration - - return { duration, isLive: video.isLive } -} - -const getCachedVideoDuration = memoizee(getVideoDuration, { - promise: true, - max: MEMOIZE_LENGTH.VIDEO_DURATION, - maxAge: MEMOIZE_TTL.VIDEO_DURATION -}) - -// --------------------------------------------------------------------------- - -async function addVideoJobsAfterUpdate (options: { - video: MVideoFullLight - isNewVideo: boolean - - nameChanged: boolean - oldPrivacy: VideoPrivacy -}) { - const { video, nameChanged, oldPrivacy, isNewVideo } = options - const jobs: CreateJobArgument[] = [] - - const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy) - - if (!video.isLive && (nameChanged || filePathChanged)) { - for (const file of (video.VideoFiles || [])) { - const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } - - jobs.push({ type: 'manage-video-torrent', payload }) - } - - const hls = video.getHLSPlaylist() - - for (const file of (hls?.VideoFiles || [])) { - const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } - - jobs.push({ type: 'manage-video-torrent', payload }) - } - } - - jobs.push({ - type: 'federate-video', - payload: { - videoUUID: video.uuid, - isNewVideo - } - }) - - const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy) - - if (wasConfidentialVideo) { - jobs.push({ - type: 'notify', - payload: { - action: 'new-video', - videoUUID: video.uuid - } - }) - } - - return JobQueue.Instance.createSequentialJobFlow(...jobs) -} - -// --------------------------------------------------------------------------- - -export { - buildLocalVideoFromReq, - buildVideoThumbnailsFromReq, - setVideoTags, - buildMoveToObjectStorageJob, - addVideoJobsAfterUpdate, - getCachedVideoDuration -} diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts deleted file mode 100644 index 139471183..000000000 --- a/server/lib/views/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-viewer-counters' -export * from './video-viewer-stats' -export * from './video-views' diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts deleted file mode 100644 index f5b83130e..000000000 --- a/server/lib/views/shared/video-viewer-counters.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { VIEW_LIFETIME } from '@server/initializers/constants' -import { sendView } from '@server/lib/activitypub/send/send-view' -import { PeerTubeSocket } from '@server/lib/peertube-socket' -import { getServerActor } from '@server/models/application/application' -import { VideoModel } from '@server/models/video/video' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { buildUUID, sha256 } from '@shared/extra-utils' - -const lTags = loggerTagsFactory('views') - -export type ViewerScope = 'local' | 'remote' -export type VideoScope = 'local' | 'remote' - -type Viewer = { - expires: number - id: string - viewerScope: ViewerScope - videoScope: VideoScope - lastFederation?: number -} - -export class VideoViewerCounters { - - // expires is new Date().getTime() - private readonly viewersPerVideo = new Map() - private readonly idToViewer = new Map() - - private readonly salt = buildUUID() - - private processingViewerCounters = false - - constructor () { - setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER) - } - - // --------------------------------------------------------------------------- - - async addLocalViewer (options: { - video: MVideoImmutable - ip: string - }) { - const { video, ip } = options - - logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) }) - - const viewerId = this.generateViewerId(ip, video.uuid) - const viewer = this.idToViewer.get(viewerId) - - if (viewer) { - viewer.expires = this.buildViewerExpireTime() - await this.federateViewerIfNeeded(video, viewer) - - return false - } - - const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' }) - await this.federateViewerIfNeeded(video, newViewer) - - return true - } - - async addRemoteViewer (options: { - video: MVideo - viewerId: string - viewerExpires: Date - }) { - const { video, viewerExpires, viewerId } = options - - logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) - - await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' }) - - return true - } - - // --------------------------------------------------------------------------- - - getTotalViewers (options: { - viewerScope: ViewerScope - videoScope: VideoScope - }) { - let total = 0 - - for (const viewers of this.viewersPerVideo.values()) { - total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length - } - - return total - } - - getViewers (video: MVideo) { - const viewers = this.viewersPerVideo.get(video.id) - if (!viewers) return 0 - - return viewers.length - } - - buildViewerExpireTime () { - return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER - } - - // --------------------------------------------------------------------------- - - private async addViewerToVideo (options: { - video: MVideoImmutable - viewerId: string - viewerScope: ViewerScope - viewerExpires?: Date - }) { - const { video, viewerExpires, viewerId, viewerScope } = options - - let watchers = this.viewersPerVideo.get(video.id) - - if (!watchers) { - watchers = [] - this.viewersPerVideo.set(video.id, watchers) - } - - const expires = viewerExpires - ? viewerExpires.getTime() - : this.buildViewerExpireTime() - - const videoScope: VideoScope = video.remote - ? 'remote' - : 'local' - - const viewer = { id: viewerId, expires, videoScope, viewerScope } - watchers.push(viewer) - - this.idToViewer.set(viewerId, viewer) - - await this.notifyClients(video.id, watchers.length) - - return viewer - } - - private async cleanViewerCounters () { - if (this.processingViewerCounters) return - this.processingViewerCounters = true - - if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags()) - - try { - for (const videoId of this.viewersPerVideo.keys()) { - const notBefore = new Date().getTime() - - const viewers = this.viewersPerVideo.get(videoId) - - // Only keep not expired viewers - const newViewers: Viewer[] = [] - - // Filter new viewers - for (const viewer of viewers) { - if (viewer.expires > notBefore) { - newViewers.push(viewer) - } else { - this.idToViewer.delete(viewer.id) - } - } - - if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) - else this.viewersPerVideo.set(videoId, newViewers) - - await this.notifyClients(videoId, newViewers.length) - } - } catch (err) { - logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) - } - - this.processingViewerCounters = false - } - - private async notifyClients (videoId: string | number, viewersLength: number) { - const video = await VideoModel.loadImmutableAttributes(videoId) - if (!video) return - - PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) - - logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) - } - - private generateViewerId (ip: string, videoUUID: string) { - return sha256(this.salt + '-' + ip + '-' + videoUUID) - } - - private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) { - // Federate the viewer if it's been a "long" time we did not - const now = new Date().getTime() - const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) - - if (viewer.lastFederation && viewer.lastFederation > federationLimit) return - - await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id }) - viewer.lastFederation = now - } -} diff --git a/server/lib/views/shared/video-viewer-stats.ts b/server/lib/views/shared/video-viewer-stats.ts deleted file mode 100644 index ebd963e59..000000000 --- a/server/lib/views/shared/video-viewer-stats.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { GeoIP } from '@server/helpers/geo-ip' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { sendCreateWatchAction } from '@server/lib/activitypub/send' -import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url' -import { Redis } from '@server/lib/redis' -import { VideoModel } from '@server/models/video/video' -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { VideoViewEvent } from '@shared/models' - -const lTags = loggerTagsFactory('views') - -type LocalViewerStats = { - firstUpdated: number // Date.getTime() - lastUpdated: number // Date.getTime() - - watchSections: { - start: number - end: number - }[] - - watchTime: number - - country: string - - videoId: number -} - -export class VideoViewerStats { - private processingViewersStats = false - - constructor () { - setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) - } - - // --------------------------------------------------------------------------- - - async addLocalViewer (options: { - video: MVideoImmutable - currentTime: number - ip: string - viewEvent?: VideoViewEvent - }) { - const { video, ip, viewEvent, currentTime } = options - - logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) - - return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) - } - - // --------------------------------------------------------------------------- - - async getWatchTime (videoId: number, ip: string) { - const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) - - return stats?.watchTime || 0 - } - - // --------------------------------------------------------------------------- - - private async updateLocalViewerStats (options: { - video: MVideoImmutable - ip: string - currentTime: number - viewEvent?: VideoViewEvent - }) { - const { video, ip, viewEvent, currentTime } = options - const nowMs = new Date().getTime() - - let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id }) - - if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) { - logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) }) - return - } - - if (!stats) { - const country = await GeoIP.Instance.safeCountryISOLookup(ip) - - stats = { - firstUpdated: nowMs, - lastUpdated: nowMs, - - watchSections: [], - - watchTime: 0, - - country, - videoId: video.id - } - } - - stats.lastUpdated = nowMs - - if (viewEvent === 'seek' || stats.watchSections.length === 0) { - stats.watchSections.push({ - start: currentTime, - end: currentTime - }) - } else { - const lastSection = stats.watchSections[stats.watchSections.length - 1] - - if (lastSection.start > currentTime) { - logger.debug('Invalid end watch section %d. Last start record was at %d. Starting a new section.', currentTime, lastSection.start) - - stats.watchSections.push({ - start: currentTime, - end: currentTime - }) - } else { - lastSection.end = currentTime - } - } - - stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections) - - logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) - - await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) - } - - async processViewerStats () { - if (this.processingViewersStats) return - this.processingViewersStats = true - - if (!isTestOrDevInstance()) logger.info('Processing viewer statistics.', lTags()) - - const now = new Date().getTime() - - try { - const allKeys = await Redis.Instance.listLocalVideoViewerKeys() - - for (const key of allKeys) { - const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key }) - - // Process expired stats - if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) { - continue - } - - try { - await sequelizeTypescript.transaction(async t => { - const video = await VideoModel.load(stats.videoId, t) - if (!video) return - - const statsModel = await this.saveViewerStats(video, stats, t) - - if (video.remote) { - await sendCreateWatchAction(statsModel, t) - } - }) - - await Redis.Instance.deleteLocalVideoViewersKeys(key) - } catch (err) { - logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() }) - } - } - } catch (err) { - logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) - } - - this.processingViewersStats = false - } - - private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { - const statsModel = new LocalVideoViewerModel({ - startDate: new Date(stats.firstUpdated), - endDate: new Date(stats.lastUpdated), - watchTime: stats.watchTime, - country: stats.country, - videoId: video.id - }) - - statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel) - statsModel.Video = video as VideoModel - - await statsModel.save({ transaction }) - - statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({ - localVideoViewerId: statsModel.id, - watchSections: stats.watchSections, - transaction - }) - - return statsModel - } - - private buildWatchTimeFromSections (sections: { start: number, end: number }[]) { - return sections.reduce((p, current) => p + (current.end - current.start), 0) - } -} diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts deleted file mode 100644 index e563287e1..000000000 --- a/server/lib/views/shared/video-views.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { sendView } from '@server/lib/activitypub/send/send-view' -import { getCachedVideoDuration } from '@server/lib/video' -import { getServerActor } from '@server/models/application/application' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { buildUUID } from '@shared/extra-utils' -import { Redis } from '../../redis' - -const lTags = loggerTagsFactory('views') - -export class VideoViews { - - async addLocalView (options: { - video: MVideoImmutable - ip: string - watchTime: number - }) { - const { video, ip, watchTime } = options - - logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) - - if (!await this.hasEnoughWatchTime(video, watchTime)) return false - - const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) - if (viewExists) return false - - await Redis.Instance.setIPVideoView(ip, video.uuid) - - await this.addView(video) - - await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() }) - - return true - } - - async addRemoteView (options: { - video: MVideo - }) { - const { video } = options - - logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) }) - - await this.addView(video) - - return true - } - - // --------------------------------------------------------------------------- - - private async addView (video: MVideoImmutable) { - const promises: Promise[] = [] - - if (video.isOwned()) { - promises.push(Redis.Instance.addLocalVideoView(video.id)) - } - - promises.push(Redis.Instance.addVideoViewStats(video.id)) - - await Promise.all(promises) - } - - private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) { - const { duration, isLive } = await getCachedVideoDuration(video.id) - - if (isLive || duration >= 30) return watchTime >= 30 - - // Check more than 50% of the video is watched - return duration / watchTime < 2 - } -} diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts deleted file mode 100644 index c088dad5e..000000000 --- a/server/lib/views/video-views-manager.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { VideoViewEvent } from '@shared/models' -import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared' - -/** - * If processing a local view: - * - We update viewer information (segments watched, watch time etc) - * - We add +1 to video viewers counter if this is a new viewer - * - We add +1 to video views counter if this is a new view and if the user watched enough seconds - * - We send AP message to notify about this viewer and this view - * - We update last video time for the user if authenticated - * - * If processing a remote view: - * - We add +1 to video viewers counter - * - We add +1 to video views counter - * - * A viewer is a someone that watched one or multiple sections of a video - * A viewer that watched only a few seconds of a video may not increment the video views counter - * Viewers statistics are sent to origin instance using the `WatchAction` ActivityPub object - * - */ - -const lTags = loggerTagsFactory('views') - -export class VideoViewsManager { - - private static instance: VideoViewsManager - - private videoViewerStats: VideoViewerStats - private videoViewerCounters: VideoViewerCounters - private videoViews: VideoViews - - private constructor () { - } - - init () { - this.videoViewerStats = new VideoViewerStats() - this.videoViewerCounters = new VideoViewerCounters() - this.videoViews = new VideoViews() - } - - async processLocalView (options: { - video: MVideoImmutable - currentTime: number - ip: string | null - viewEvent?: VideoViewEvent - }) { - const { video, ip, viewEvent, currentTime } = options - - logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) - - await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime }) - - const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip }) - - // Do it after added local viewer to fetch updated information - const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip) - - const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) - - return { successView, successViewer } - } - - async processRemoteView (options: { - video: MVideo - viewerId: string | null - viewerExpires?: Date - }) { - const { video, viewerId, viewerExpires } = options - - logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() }) - - if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires }) - else await this.videoViews.addRemoteView({ video }) - } - - getViewers (video: MVideo) { - return this.videoViewerCounters.getViewers(video) - } - - getTotalViewers (options: { - viewerScope: ViewerScope - videoScope: VideoScope - }) { - return this.videoViewerCounters.getTotalViewers(options) - } - - buildViewerExpireTime () { - return this.videoViewerCounters.buildViewerExpireTime() - } - - processViewerStats () { - return this.videoViewerStats.processViewerStats() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts deleted file mode 100644 index 48b6c682b..000000000 --- a/server/lib/worker/parent-process.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { join } from 'path' -import Piscina from 'piscina' -import { processImage } from '@server/helpers/image-utils' -import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants' -import { httpBroadcast } from './workers/http-broadcast' -import { downloadImage } from './workers/image-downloader' - -let downloadImageWorker: Piscina - -function downloadImageFromWorker (options: Parameters[0]): Promise> { - if (!downloadImageWorker) { - downloadImageWorker = new Piscina({ - filename: join(__dirname, 'workers', 'image-downloader.js'), - concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY, - maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS - }) - } - - return downloadImageWorker.run(options) -} - -// --------------------------------------------------------------------------- - -let processImageWorker: Piscina - -function processImageFromWorker (options: Parameters[0]): Promise> { - if (!processImageWorker) { - processImageWorker = new Piscina({ - filename: join(__dirname, 'workers', 'image-processor.js'), - concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY, - maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS - }) - } - - return processImageWorker.run(options) -} - -// --------------------------------------------------------------------------- - -let parallelHTTPBroadcastWorker: Piscina - -function parallelHTTPBroadcastFromWorker (options: Parameters[0]): Promise> { - if (!parallelHTTPBroadcastWorker) { - parallelHTTPBroadcastWorker = new Piscina({ - filename: join(__dirname, 'workers', 'http-broadcast.js'), - // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs - concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'], - maxThreads: 1 - }) - } - - return parallelHTTPBroadcastWorker.run(options) -} - -// --------------------------------------------------------------------------- - -let sequentialHTTPBroadcastWorker: Piscina - -function sequentialHTTPBroadcastFromWorker (options: Parameters[0]): Promise> { - if (!sequentialHTTPBroadcastWorker) { - sequentialHTTPBroadcastWorker = new Piscina({ - filename: join(__dirname, 'workers', 'http-broadcast.js'), - // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs - concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'], - maxThreads: 1 - }) - } - - return sequentialHTTPBroadcastWorker.run(options) -} - -export { - downloadImageFromWorker, - processImageFromWorker, - parallelHTTPBroadcastFromWorker, - sequentialHTTPBroadcastFromWorker -} diff --git a/server/lib/worker/workers/http-broadcast.ts b/server/lib/worker/workers/http-broadcast.ts deleted file mode 100644 index 8c9c6b8ca..000000000 --- a/server/lib/worker/workers/http-broadcast.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { map } from 'bluebird' -import { logger } from '@server/helpers/logger' -import { doRequest, PeerTubeRequestOptions } from '@server/helpers/requests' -import { BROADCAST_CONCURRENCY } from '@server/initializers/constants' - -async function httpBroadcast (payload: { - uris: string[] - requestOptions: PeerTubeRequestOptions -}) { - const { uris, requestOptions } = payload - - const badUrls: string[] = [] - const goodUrls: string[] = [] - - await map(uris, async uri => { - try { - await doRequest(uri, requestOptions) - goodUrls.push(uri) - } catch (err) { - logger.debug('HTTP broadcast to %s failed.', uri, { err }) - badUrls.push(uri) - } - }, { concurrency: BROADCAST_CONCURRENCY }) - - return { goodUrls, badUrls } -} - -module.exports = httpBroadcast - -export { - httpBroadcast -} diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts deleted file mode 100644 index 209594589..000000000 --- a/server/lib/worker/workers/image-downloader.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { processImage } from '@server/helpers/image-utils' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' - -async function downloadImage (options: { - url: string - destDir: string - destName: string - size: { width: number, height: number } -}) { - const { url, destDir, destName, size } = options - - const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) - await doRequestAndSaveToFile(url, tmpPath) - - const destPath = join(destDir, destName) - - try { - await processImage({ path: tmpPath, destination: destPath, newSize: size }) - } catch (err) { - await remove(tmpPath) - - throw err - } - - return destPath -} - -module.exports = downloadImage - -export { - downloadImage -} diff --git a/server/lib/worker/workers/image-processor.ts b/server/lib/worker/workers/image-processor.ts deleted file mode 100644 index 0ab41a5a0..000000000 --- a/server/lib/worker/workers/image-processor.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { processImage } from '@server/helpers/image-utils' - -module.exports = processImage - -export { - processImage -} -- cgit v1.2.3