shared/server-commands/server/server.ts

   1 import { ChildProcess, fork } from 'child_process'
   2 import { copy } from 'fs-extra'
   3 import { join } from 'path'
   4 import { parallelTests, randomInt, root } from '@shared/core-utils'
   5 import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@shared/models'
   6 import { BulkCommand } from '../bulk'
   7 import { CLICommand } from '../cli'
   8 import { CustomPagesCommand } from '../custom-pages'
   9 import { FeedCommand } from '../feeds'
  10 import { LogsCommand } from '../logs'
  11 import { SQLCommand } from '../miscs'
  12 import { AbusesCommand } from '../moderation'
  13 import { OverviewsCommand } from '../overviews'
  14 import { SearchCommand } from '../search'
  15 import { SocketIOCommand } from '../socket'
  16 import {
  17   AccountsCommand,
  18   BlocklistCommand,
  19   LoginCommand,
  20   NotificationsCommand,
  21   SubscriptionsCommand,
  22   TwoFactorCommand,
  23   UsersCommand
  24 } from '../users'
  25 import {
  26   BlacklistCommand,
  27   CaptionsCommand,
  28   ChangeOwnershipCommand,
  29   ChannelsCommand,
  30   ChannelSyncsCommand,
  31   HistoryCommand,
  32   ImportsCommand,
  33   LiveCommand,
  34   PlaylistsCommand,
  35   ServicesCommand,
  36   StreamingPlaylistsCommand,
  37   VideosCommand,
  38   VideoStudioCommand,
  39   VideoTokenCommand,
  40   ViewsCommand
  41 } from '../videos'
  42 import { CommentsCommand } from '../videos/comments-command'
  43 import { VideoStatsCommand } from '../videos/video-stats-command'
  44 import { ConfigCommand } from './config-command'
  45 import { ContactFormCommand } from './contact-form-command'
  46 import { DebugCommand } from './debug-command'
  47 import { FollowsCommand } from './follows-command'
  48 import { JobsCommand } from './jobs-command'
  49 import { MetricsCommand } from './metrics-command'
  50 import { ObjectStorageCommand } from './object-storage-command'
  51 import { PluginsCommand } from './plugins-command'
  52 import { RedundancyCommand } from './redundancy-command'
  53 import { ServersCommand } from './servers-command'
  54 import { StatsCommand } from './stats-command'
  55
  56 export type RunServerOptions = {
  57   hideLogs?: boolean
  58   nodeArgs?: string[]
  59   peertubeArgs?: string[]
  60   env?: { [ id: string ]: string }
  61 }
  62
  63 export class PeerTubeServer {
  64   app?: ChildProcess
  65
  66   url: string
  67   host?: string
  68   hostname?: string
  69   port?: number
  70
  71   rtmpPort?: number
  72   rtmpsPort?: number
  73
  74   parallel?: boolean
  75   internalServerNumber: number
  76
  77   serverNumber?: number
  78   customConfigFile?: string
  79
  80   store?: {
  81     client?: {
  82       id?: string
  83       secret?: string
  84     }
  85
  86     user?: {
  87       username: string
  88       password: string
  89       email?: string
  90     }
  91
  92     channel?: VideoChannel
  93     videoChannelSync?: Partial<VideoChannelSync>
  94
  95     video?: Video
  96     videoCreated?: VideoCreateResult
  97     videoDetails?: VideoDetails
  98
  99     videos?: { id: number, uuid: string }[]
 100   }
 101
 102   accessToken?: string
 103   refreshToken?: string
 104
 105   bulk?: BulkCommand
 106   cli?: CLICommand
 107   customPage?: CustomPagesCommand
 108   feed?: FeedCommand
 109   logs?: LogsCommand
 110   abuses?: AbusesCommand
 111   overviews?: OverviewsCommand
 112   search?: SearchCommand
 113   contactForm?: ContactFormCommand
 114   debug?: DebugCommand
 115   follows?: FollowsCommand
 116   jobs?: JobsCommand
 117   metrics?: MetricsCommand
 118   plugins?: PluginsCommand
 119   redundancy?: RedundancyCommand
 120   stats?: StatsCommand
 121   config?: ConfigCommand
 122   socketIO?: SocketIOCommand
 123   accounts?: AccountsCommand
 124   blocklist?: BlocklistCommand
 125   subscriptions?: SubscriptionsCommand
 126   live?: LiveCommand
 127   services?: ServicesCommand
 128   blacklist?: BlacklistCommand
 129   captions?: CaptionsCommand
 130   changeOwnership?: ChangeOwnershipCommand
 131   playlists?: PlaylistsCommand
 132   history?: HistoryCommand
 133   imports?: ImportsCommand
 134   channelSyncs?: ChannelSyncsCommand
 135   streamingPlaylists?: StreamingPlaylistsCommand
 136   channels?: ChannelsCommand
 137   comments?: CommentsCommand
 138   sql?: SQLCommand
 139   notifications?: NotificationsCommand
 140   servers?: ServersCommand
 141   login?: LoginCommand
 142   users?: UsersCommand
 143   objectStorage?: ObjectStorageCommand
 144   videoStudio?: VideoStudioCommand
 145   videos?: VideosCommand
 146   videoStats?: VideoStatsCommand
 147   views?: ViewsCommand
 148   twoFactor?: TwoFactorCommand
 149   videoToken?: VideoTokenCommand
 150
 151   constructor (options: { serverNumber: number } | { url: string }) {
 152     if ((options as any).url) {
 153       this.setUrl((options as any).url)
 154     } else {
 155       this.setServerNumber((options as any).serverNumber)
 156     }
 157
 158     this.store = {
 159       client: {
 160         id: null,
 161         secret: null
 162       },
 163       user: {
 164         username: null,
 165         password: null
 166       }
 167     }
 168
 169     this.assignCommands()
 170   }
 171
 172   setServerNumber (serverNumber: number) {
 173     this.serverNumber = serverNumber
 174
 175     this.parallel = parallelTests()
 176
 177     this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber
 178     this.rtmpPort = this.parallel ? this.randomRTMP() : 1936
 179     this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937
 180     this.port = 9000 + this.internalServerNumber
 181
 182     this.url = `http://localhost:${this.port}`
 183     this.host = `localhost:${this.port}`
 184     this.hostname = 'localhost'
 185   }
 186
 187   setUrl (url: string) {
 188     const parsed = new URL(url)
 189
 190     this.url = url
 191     this.host = parsed.host
 192     this.hostname = parsed.hostname
 193     this.port = parseInt(parsed.port)
 194   }
 195
 196   getDirectoryPath (directoryName: string) {
 197     const testDirectory = 'test' + this.internalServerNumber
 198
 199     return join(root(), testDirectory, directoryName)
 200   }
 201
 202   async flushAndRun (configOverride?: Object, options: RunServerOptions = {}) {
 203     await ServersCommand.flushTests(this.internalServerNumber)
 204
 205     return this.run(configOverride, options)
 206   }
 207
 208   async run (configOverrideArg?: any, options: RunServerOptions = {}) {
 209     // These actions are async so we need to be sure that they have both been done
 210     const serverRunString = {
 211       'HTTP server listening': false
 212     }
 213     const key = 'Database peertube_test' + this.internalServerNumber + ' is ready'
 214     serverRunString[key] = false
 215
 216     const regexps = {
 217       client_id: 'Client id: (.+)',
 218       client_secret: 'Client secret: (.+)',
 219       user_username: 'Username: (.+)',
 220       user_password: 'User password: (.+)'
 221     }
 222
 223     await this.assignCustomConfigFile()
 224
 225     const configOverride = this.buildConfigOverride()
 226
 227     if (configOverrideArg !== undefined) {
 228       Object.assign(configOverride, configOverrideArg)
 229     }
 230
 231     // Share the environment
 232     const env = Object.create(process.env)
 233     env['NODE_ENV'] = 'test'
 234     env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
 235     env['NODE_CONFIG'] = JSON.stringify(configOverride)
 236
 237     if (options.env) {
 238       Object.assign(env, options.env)
 239     }
 240
 241     const execArgv = options.nodeArgs || []
 242     // FIXME: too slow :/
 243     // execArgv.push('--enable-source-maps')
 244
 245     const forkOptions = {
 246       silent: true,
 247       env,
 248       detached: true,
 249       execArgv
 250     }
 251
 252     const peertubeArgs = options.peertubeArgs || []
 253
 254     return new Promise<void>((res, rej) => {
 255       const self = this
 256       let aggregatedLogs = ''
 257
 258       this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions)
 259
 260       const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs))
 261       const onParentExit = () => {
 262         if (!this.app || !this.app.pid) return
 263
 264         try {
 265           process.kill(self.app.pid)
 266         } catch { /* empty */ }
 267       }
 268
 269       this.app.on('exit', onPeerTubeExit)
 270       process.on('exit', onParentExit)
 271
 272       this.app.stdout.on('data', function onStdout (data) {
 273         let dontContinue = false
 274
 275         const log: string = data.toString()
 276         aggregatedLogs += log
 277
 278         // Capture things if we want to
 279         for (const key of Object.keys(regexps)) {
 280           const regexp = regexps[key]
 281           const matches = log.match(regexp)
 282           if (matches !== null) {
 283             if (key === 'client_id') self.store.client.id = matches[1]
 284             else if (key === 'client_secret') self.store.client.secret = matches[1]
 285             else if (key === 'user_username') self.store.user.username = matches[1]
 286             else if (key === 'user_password') self.store.user.password = matches[1]
 287           }
 288         }
 289
 290         // Check if all required sentences are here
 291         for (const key of Object.keys(serverRunString)) {
 292           if (log.includes(key)) serverRunString[key] = true
 293           if (serverRunString[key] === false) dontContinue = true
 294         }
 295
 296         // If no, there is maybe one thing not already initialized (client/user credentials generation...)
 297         if (dontContinue === true) return
 298
 299         if (options.hideLogs === false) {
 300           console.log(log)
 301         } else {
 302           process.removeListener('exit', onParentExit)
 303           self.app.stdout.removeListener('data', onStdout)
 304           self.app.removeListener('exit', onPeerTubeExit)
 305         }
 306
 307         res()
 308       })
 309     })
 310   }
 311
 312   async kill () {
 313     if (!this.app) return
 314
 315     await this.sql.cleanup()
 316
 317     process.kill(-this.app.pid)
 318
 319     this.app = null
 320   }
 321
 322   private randomServer () {
 323     const low = 10
 324     const high = 10000
 325
 326     return randomInt(low, high)
 327   }
 328
 329   private randomRTMP () {
 330     const low = 1900
 331     const high = 2100
 332
 333     return randomInt(low, high)
 334   }
 335
 336   private async assignCustomConfigFile () {
 337     if (this.internalServerNumber === this.serverNumber) return
 338
 339     const basePath = join(root(), 'config')
 340
 341     const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`)
 342     await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile)
 343
 344     this.customConfigFile = tmpConfigFile
 345   }
 346
 347   private buildConfigOverride () {
 348     if (!this.parallel) return {}
 349
 350     return {
 351       listen: {
 352         port: this.port
 353       },
 354       webserver: {
 355         port: this.port
 356       },
 357       database: {
 358         suffix: '_test' + this.internalServerNumber
 359       },
 360       storage: {
 361         tmp: this.getDirectoryPath('tmp') + '/',
 362         bin: this.getDirectoryPath('bin') + '/',
 363         avatars: this.getDirectoryPath('avatars') + '/',
 364         videos: this.getDirectoryPath('videos') + '/',
 365         streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/',
 366         redundancy: this.getDirectoryPath('redundancy') + '/',
 367         logs: this.getDirectoryPath('logs') + '/',
 368         previews: this.getDirectoryPath('previews') + '/',
 369         thumbnails: this.getDirectoryPath('thumbnails') + '/',
 370         torrents: this.getDirectoryPath('torrents') + '/',
 371         captions: this.getDirectoryPath('captions') + '/',
 372         cache: this.getDirectoryPath('cache') + '/',
 373         plugins: this.getDirectoryPath('plugins') + '/',
 374         well_known: this.getDirectoryPath('well-known') + '/'
 375       },
 376       admin: {
 377         email: `admin${this.internalServerNumber}@example.com`
 378       },
 379       live: {
 380         rtmp: {
 381           port: this.rtmpPort
 382         }
 383       }
 384     }
 385   }
 386
 387   private assignCommands () {
 388     this.bulk = new BulkCommand(this)
 389     this.cli = new CLICommand(this)
 390     this.customPage = new CustomPagesCommand(this)
 391     this.feed = new FeedCommand(this)
 392     this.logs = new LogsCommand(this)
 393     this.abuses = new AbusesCommand(this)
 394     this.overviews = new OverviewsCommand(this)
 395     this.search = new SearchCommand(this)
 396     this.contactForm = new ContactFormCommand(this)
 397     this.debug = new DebugCommand(this)
 398     this.follows = new FollowsCommand(this)
 399     this.jobs = new JobsCommand(this)
 400     this.metrics = new MetricsCommand(this)
 401     this.plugins = new PluginsCommand(this)
 402     this.redundancy = new RedundancyCommand(this)
 403     this.stats = new StatsCommand(this)
 404     this.config = new ConfigCommand(this)
 405     this.socketIO = new SocketIOCommand(this)
 406     this.accounts = new AccountsCommand(this)
 407     this.blocklist = new BlocklistCommand(this)
 408     this.subscriptions = new SubscriptionsCommand(this)
 409     this.live = new LiveCommand(this)
 410     this.services = new ServicesCommand(this)
 411     this.blacklist = new BlacklistCommand(this)
 412     this.captions = new CaptionsCommand(this)
 413     this.changeOwnership = new ChangeOwnershipCommand(this)
 414     this.playlists = new PlaylistsCommand(this)
 415     this.history = new HistoryCommand(this)
 416     this.imports = new ImportsCommand(this)
 417     this.channelSyncs = new ChannelSyncsCommand(this)
 418     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
 419     this.channels = new ChannelsCommand(this)
 420     this.comments = new CommentsCommand(this)
 421     this.sql = new SQLCommand(this)
 422     this.notifications = new NotificationsCommand(this)
 423     this.servers = new ServersCommand(this)
 424     this.login = new LoginCommand(this)
 425     this.users = new UsersCommand(this)
 426     this.videos = new VideosCommand(this)
 427     this.objectStorage = new ObjectStorageCommand(this)
 428     this.videoStudio = new VideoStudioCommand(this)
 429     this.videoStats = new VideoStatsCommand(this)
 430     this.views = new ViewsCommand(this)
 431     this.twoFactor = new TwoFactorCommand(this)
 432     this.videoToken = new VideoTokenCommand(this)
 433   }
 434 }