shared/server-commands/server/server.ts

   1 import { ChildProcess, fork } from 'child_process'
   2 import { copy } from 'fs-extra'
   3 import { join } from 'path'
   4 import { parallelTests, randomInt, root } from '@shared/core-utils'
   5 import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@shared/models'
   6 import { BulkCommand } from '../bulk'
   7 import { CLICommand } from '../cli'
   8 import { CustomPagesCommand } from '../custom-pages'
   9 import { FeedCommand } from '../feeds'
  10 import { LogsCommand } from '../logs'
  11 import { SQLCommand } from '../miscs'
  12 import { AbusesCommand } from '../moderation'
  13 import { OverviewsCommand } from '../overviews'
  14 import { SearchCommand } from '../search'
  15 import { SocketIOCommand } from '../socket'
  16 import {
  17   AccountsCommand,
  18   BlocklistCommand,
  19   LoginCommand,
  20   NotificationsCommand,
  21   RegistrationsCommand,
  22   SubscriptionsCommand,
  23   TwoFactorCommand,
  24   UsersCommand
  25 } from '../users'
  26 import {
  27   BlacklistCommand,
  28   CaptionsCommand,
  29   ChangeOwnershipCommand,
  30   ChannelsCommand,
  31   ChannelSyncsCommand,
  32   HistoryCommand,
  33   ImportsCommand,
  34   LiveCommand,
  35   PlaylistsCommand,
  36   ServicesCommand,
  37   StreamingPlaylistsCommand,
  38   VideosCommand,
  39   VideoStudioCommand,
  40   VideoTokenCommand,
  41   ViewsCommand
  42 } from '../videos'
  43 import { CommentsCommand } from '../videos/comments-command'
  44 import { VideoStatsCommand } from '../videos/video-stats-command'
  45 import { ConfigCommand } from './config-command'
  46 import { ContactFormCommand } from './contact-form-command'
  47 import { DebugCommand } from './debug-command'
  48 import { FollowsCommand } from './follows-command'
  49 import { JobsCommand } from './jobs-command'
  50 import { MetricsCommand } from './metrics-command'
  51 import { ObjectStorageCommand } from './object-storage-command'
  52 import { PluginsCommand } from './plugins-command'
  53 import { RedundancyCommand } from './redundancy-command'
  54 import { ServersCommand } from './servers-command'
  55 import { StatsCommand } from './stats-command'
  56
  57 export type RunServerOptions = {
  58   hideLogs?: boolean
  59   nodeArgs?: string[]
  60   peertubeArgs?: string[]
  61   env?: { [ id: string ]: string }
  62 }
  63
  64 export class PeerTubeServer {
  65   app?: ChildProcess
  66
  67   url: string
  68   host?: string
  69   hostname?: string
  70   port?: number
  71
  72   rtmpPort?: number
  73   rtmpsPort?: number
  74
  75   parallel?: boolean
  76   internalServerNumber: number
  77
  78   serverNumber?: number
  79   customConfigFile?: string
  80
  81   store?: {
  82     client?: {
  83       id?: string
  84       secret?: string
  85     }
  86
  87     user?: {
  88       username: string
  89       password: string
  90       email?: string
  91     }
  92
  93     channel?: VideoChannel
  94     videoChannelSync?: Partial<VideoChannelSync>
  95
  96     video?: Video
  97     videoCreated?: VideoCreateResult
  98     videoDetails?: VideoDetails
  99
 100     videos?: { id: number, uuid: string }[]
 101   }
 102
 103   accessToken?: string
 104   refreshToken?: string
 105
 106   bulk?: BulkCommand
 107   cli?: CLICommand
 108   customPage?: CustomPagesCommand
 109   feed?: FeedCommand
 110   logs?: LogsCommand
 111   abuses?: AbusesCommand
 112   overviews?: OverviewsCommand
 113   search?: SearchCommand
 114   contactForm?: ContactFormCommand
 115   debug?: DebugCommand
 116   follows?: FollowsCommand
 117   jobs?: JobsCommand
 118   metrics?: MetricsCommand
 119   plugins?: PluginsCommand
 120   redundancy?: RedundancyCommand
 121   stats?: StatsCommand
 122   config?: ConfigCommand
 123   socketIO?: SocketIOCommand
 124   accounts?: AccountsCommand
 125   blocklist?: BlocklistCommand
 126   subscriptions?: SubscriptionsCommand
 127   live?: LiveCommand
 128   services?: ServicesCommand
 129   blacklist?: BlacklistCommand
 130   captions?: CaptionsCommand
 131   changeOwnership?: ChangeOwnershipCommand
 132   playlists?: PlaylistsCommand
 133   history?: HistoryCommand
 134   imports?: ImportsCommand
 135   channelSyncs?: ChannelSyncsCommand
 136   streamingPlaylists?: StreamingPlaylistsCommand
 137   channels?: ChannelsCommand
 138   comments?: CommentsCommand
 139   sql?: SQLCommand
 140   notifications?: NotificationsCommand
 141   servers?: ServersCommand
 142   login?: LoginCommand
 143   users?: UsersCommand
 144   objectStorage?: ObjectStorageCommand
 145   videoStudio?: VideoStudioCommand
 146   videos?: VideosCommand
 147   videoStats?: VideoStatsCommand
 148   views?: ViewsCommand
 149   twoFactor?: TwoFactorCommand
 150   videoToken?: VideoTokenCommand
 151   registrations?: RegistrationsCommand
 152
 153   constructor (options: { serverNumber: number } | { url: string }) {
 154     if ((options as any).url) {
 155       this.setUrl((options as any).url)
 156     } else {
 157       this.setServerNumber((options as any).serverNumber)
 158     }
 159
 160     this.store = {
 161       client: {
 162         id: null,
 163         secret: null
 164       },
 165       user: {
 166         username: null,
 167         password: null
 168       }
 169     }
 170
 171     this.assignCommands()
 172   }
 173
 174   setServerNumber (serverNumber: number) {
 175     this.serverNumber = serverNumber
 176
 177     this.parallel = parallelTests()
 178
 179     this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber
 180     this.rtmpPort = this.parallel ? this.randomRTMP() : 1936
 181     this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937
 182     this.port = 9000 + this.internalServerNumber
 183
 184     this.url = `http://127.0.0.1:${this.port}`
 185     this.host = `127.0.0.1:${this.port}`
 186     this.hostname = '127.0.0.1'
 187   }
 188
 189   setUrl (url: string) {
 190     const parsed = new URL(url)
 191
 192     this.url = url
 193     this.host = parsed.host
 194     this.hostname = parsed.hostname
 195     this.port = parseInt(parsed.port)
 196   }
 197
 198   getDirectoryPath (directoryName: string) {
 199     const testDirectory = 'test' + this.internalServerNumber
 200
 201     return join(root(), testDirectory, directoryName)
 202   }
 203
 204   async flushAndRun (configOverride?: object, options: RunServerOptions = {}) {
 205     await ServersCommand.flushTests(this.internalServerNumber)
 206
 207     return this.run(configOverride, options)
 208   }
 209
 210   async run (configOverrideArg?: any, options: RunServerOptions = {}) {
 211     // These actions are async so we need to be sure that they have both been done
 212     const serverRunString = {
 213       'HTTP server listening': false
 214     }
 215     const key = 'Database peertube_test' + this.internalServerNumber + ' is ready'
 216     serverRunString[key] = false
 217
 218     const regexps = {
 219       client_id: 'Client id: (.+)',
 220       client_secret: 'Client secret: (.+)',
 221       user_username: 'Username: (.+)',
 222       user_password: 'User password: (.+)'
 223     }
 224
 225     await this.assignCustomConfigFile()
 226
 227     const configOverride = this.buildConfigOverride()
 228
 229     if (configOverrideArg !== undefined) {
 230       Object.assign(configOverride, configOverrideArg)
 231     }
 232
 233     // Share the environment
 234     const env = Object.create(process.env)
 235     env['NODE_ENV'] = 'test'
 236     env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
 237     env['NODE_CONFIG'] = JSON.stringify(configOverride)
 238
 239     if (options.env) {
 240       Object.assign(env, options.env)
 241     }
 242
 243     const execArgv = options.nodeArgs || []
 244     // FIXME: too slow :/
 245     // execArgv.push('--enable-source-maps')
 246
 247     const forkOptions = {
 248       silent: true,
 249       env,
 250       detached: true,
 251       execArgv
 252     }
 253
 254     const peertubeArgs = options.peertubeArgs || []
 255
 256     return new Promise<void>((res, rej) => {
 257       const self = this
 258       let aggregatedLogs = ''
 259
 260       this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions)
 261
 262       const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs))
 263       const onParentExit = () => {
 264         if (!this.app?.pid) return
 265
 266         try {
 267           process.kill(self.app.pid)
 268         } catch { /* empty */ }
 269       }
 270
 271       this.app.on('exit', onPeerTubeExit)
 272       process.on('exit', onParentExit)
 273
 274       this.app.stdout.on('data', function onStdout (data) {
 275         let dontContinue = false
 276
 277         const log: string = data.toString()
 278         aggregatedLogs += log
 279
 280         // Capture things if we want to
 281         for (const key of Object.keys(regexps)) {
 282           const regexp = regexps[key]
 283           const matches = log.match(regexp)
 284           if (matches !== null) {
 285             if (key === 'client_id') self.store.client.id = matches[1]
 286             else if (key === 'client_secret') self.store.client.secret = matches[1]
 287             else if (key === 'user_username') self.store.user.username = matches[1]
 288             else if (key === 'user_password') self.store.user.password = matches[1]
 289           }
 290         }
 291
 292         // Check if all required sentences are here
 293         for (const key of Object.keys(serverRunString)) {
 294           if (log.includes(key)) serverRunString[key] = true
 295           if (serverRunString[key] === false) dontContinue = true
 296         }
 297
 298         // If no, there is maybe one thing not already initialized (client/user credentials generation...)
 299         if (dontContinue === true) return
 300
 301         if (options.hideLogs === false) {
 302           console.log(log)
 303         } else {
 304           process.removeListener('exit', onParentExit)
 305           self.app.stdout.removeListener('data', onStdout)
 306           self.app.removeListener('exit', onPeerTubeExit)
 307         }
 308
 309         res()
 310       })
 311     })
 312   }
 313
 314   async kill () {
 315     if (!this.app) return
 316
 317     await this.sql.cleanup()
 318
 319     process.kill(-this.app.pid)
 320
 321     this.app = null
 322   }
 323
 324   private randomServer () {
 325     const low = 10
 326     const high = 10000
 327
 328     return randomInt(low, high)
 329   }
 330
 331   private randomRTMP () {
 332     const low = 1900
 333     const high = 2100
 334
 335     return randomInt(low, high)
 336   }
 337
 338   private async assignCustomConfigFile () {
 339     if (this.internalServerNumber === this.serverNumber) return
 340
 341     const basePath = join(root(), 'config')
 342
 343     const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`)
 344     await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile)
 345
 346     this.customConfigFile = tmpConfigFile
 347   }
 348
 349   private buildConfigOverride () {
 350     if (!this.parallel) return {}
 351
 352     return {
 353       listen: {
 354         port: this.port
 355       },
 356       webserver: {
 357         port: this.port
 358       },
 359       database: {
 360         suffix: '_test' + this.internalServerNumber
 361       },
 362       storage: {
 363         tmp: this.getDirectoryPath('tmp') + '/',
 364         bin: this.getDirectoryPath('bin') + '/',
 365         avatars: this.getDirectoryPath('avatars') + '/',
 366         videos: this.getDirectoryPath('videos') + '/',
 367         streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/',
 368         redundancy: this.getDirectoryPath('redundancy') + '/',
 369         logs: this.getDirectoryPath('logs') + '/',
 370         previews: this.getDirectoryPath('previews') + '/',
 371         thumbnails: this.getDirectoryPath('thumbnails') + '/',
 372         torrents: this.getDirectoryPath('torrents') + '/',
 373         captions: this.getDirectoryPath('captions') + '/',
 374         cache: this.getDirectoryPath('cache') + '/',
 375         plugins: this.getDirectoryPath('plugins') + '/',
 376         well_known: this.getDirectoryPath('well-known') + '/'
 377       },
 378       admin: {
 379         email: `admin${this.internalServerNumber}@example.com`
 380       },
 381       live: {
 382         rtmp: {
 383           port: this.rtmpPort
 384         }
 385       }
 386     }
 387   }
 388
 389   private assignCommands () {
 390     this.bulk = new BulkCommand(this)
 391     this.cli = new CLICommand(this)
 392     this.customPage = new CustomPagesCommand(this)
 393     this.feed = new FeedCommand(this)
 394     this.logs = new LogsCommand(this)
 395     this.abuses = new AbusesCommand(this)
 396     this.overviews = new OverviewsCommand(this)
 397     this.search = new SearchCommand(this)
 398     this.contactForm = new ContactFormCommand(this)
 399     this.debug = new DebugCommand(this)
 400     this.follows = new FollowsCommand(this)
 401     this.jobs = new JobsCommand(this)
 402     this.metrics = new MetricsCommand(this)
 403     this.plugins = new PluginsCommand(this)
 404     this.redundancy = new RedundancyCommand(this)
 405     this.stats = new StatsCommand(this)
 406     this.config = new ConfigCommand(this)
 407     this.socketIO = new SocketIOCommand(this)
 408     this.accounts = new AccountsCommand(this)
 409     this.blocklist = new BlocklistCommand(this)
 410     this.subscriptions = new SubscriptionsCommand(this)
 411     this.live = new LiveCommand(this)
 412     this.services = new ServicesCommand(this)
 413     this.blacklist = new BlacklistCommand(this)
 414     this.captions = new CaptionsCommand(this)
 415     this.changeOwnership = new ChangeOwnershipCommand(this)
 416     this.playlists = new PlaylistsCommand(this)
 417     this.history = new HistoryCommand(this)
 418     this.imports = new ImportsCommand(this)
 419     this.channelSyncs = new ChannelSyncsCommand(this)
 420     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
 421     this.channels = new ChannelsCommand(this)
 422     this.comments = new CommentsCommand(this)
 423     this.sql = new SQLCommand(this)
 424     this.notifications = new NotificationsCommand(this)
 425     this.servers = new ServersCommand(this)
 426     this.login = new LoginCommand(this)
 427     this.users = new UsersCommand(this)
 428     this.videos = new VideosCommand(this)
 429     this.objectStorage = new ObjectStorageCommand(this)
 430     this.videoStudio = new VideoStudioCommand(this)
 431     this.videoStats = new VideoStatsCommand(this)
 432     this.views = new ViewsCommand(this)
 433     this.twoFactor = new TwoFactorCommand(this)
 434     this.videoToken = new VideoTokenCommand(this)
 435     this.registrations = new RegistrationsCommand(this)
 436   }
 437 }