shared/server-commands/server/server.ts

   1 import { ChildProcess, fork } from 'child_process'
   2 import { copy } from 'fs-extra'
   3 import { join } from 'path'
   4 import { parallelTests, randomInt, root } from '@shared/core-utils'
   5 import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@shared/models'
   6 import { BulkCommand } from '../bulk'
   7 import { CLICommand } from '../cli'
   8 import { CustomPagesCommand } from '../custom-pages'
   9 import { FeedCommand } from '../feeds'
  10 import { LogsCommand } from '../logs'
  11 import { SQLCommand } from '../miscs'
  12 import { AbusesCommand } from '../moderation'
  13 import { OverviewsCommand } from '../overviews'
  14 import { SearchCommand } from '../search'
  15 import { SocketIOCommand } from '../socket'
  16 import { AccountsCommand, BlocklistCommand, LoginCommand, NotificationsCommand, SubscriptionsCommand, UsersCommand } from '../users'
  17 import {
  18   BlacklistCommand,
  19   CaptionsCommand,
  20   ChangeOwnershipCommand,
  21   ChannelsCommand,
  22   ChannelSyncsCommand,
  23   HistoryCommand,
  24   ImportsCommand,
  25   LiveCommand,
  26   PlaylistsCommand,
  27   ServicesCommand,
  28   StreamingPlaylistsCommand,
  29   VideosCommand,
  30   VideoStudioCommand,
  31   ViewsCommand
  32 } from '../videos'
  33 import { CommentsCommand } from '../videos/comments-command'
  34 import { VideoStatsCommand } from '../videos/video-stats-command'
  35 import { ConfigCommand } from './config-command'
  36 import { ContactFormCommand } from './contact-form-command'
  37 import { DebugCommand } from './debug-command'
  38 import { FollowsCommand } from './follows-command'
  39 import { JobsCommand } from './jobs-command'
  40 import { MetricsCommand } from './metrics-command'
  41 import { ObjectStorageCommand } from './object-storage-command'
  42 import { PluginsCommand } from './plugins-command'
  43 import { RedundancyCommand } from './redundancy-command'
  44 import { ServersCommand } from './servers-command'
  45 import { StatsCommand } from './stats-command'
  46
  47 export type RunServerOptions = {
  48   hideLogs?: boolean
  49   nodeArgs?: string[]
  50   peertubeArgs?: string[]
  51   env?: { [ id: string ]: string }
  52 }
  53
  54 export class PeerTubeServer {
  55   app?: ChildProcess
  56
  57   url: string
  58   host?: string
  59   hostname?: string
  60   port?: number
  61
  62   rtmpPort?: number
  63   rtmpsPort?: number
  64
  65   parallel?: boolean
  66   internalServerNumber: number
  67
  68   serverNumber?: number
  69   customConfigFile?: string
  70
  71   store?: {
  72     client?: {
  73       id?: string
  74       secret?: string
  75     }
  76
  77     user?: {
  78       username: string
  79       password: string
  80       email?: string
  81     }
  82
  83     channel?: VideoChannel
  84     videoChannelSync?: Partial<VideoChannelSync>
  85
  86     video?: Video
  87     videoCreated?: VideoCreateResult
  88     videoDetails?: VideoDetails
  89
  90     videos?: { id: number, uuid: string }[]
  91   }
  92
  93   accessToken?: string
  94   refreshToken?: string
  95
  96   bulk?: BulkCommand
  97   cli?: CLICommand
  98   customPage?: CustomPagesCommand
  99   feed?: FeedCommand
 100   logs?: LogsCommand
 101   abuses?: AbusesCommand
 102   overviews?: OverviewsCommand
 103   search?: SearchCommand
 104   contactForm?: ContactFormCommand
 105   debug?: DebugCommand
 106   follows?: FollowsCommand
 107   jobs?: JobsCommand
 108   metrics?: MetricsCommand
 109   plugins?: PluginsCommand
 110   redundancy?: RedundancyCommand
 111   stats?: StatsCommand
 112   config?: ConfigCommand
 113   socketIO?: SocketIOCommand
 114   accounts?: AccountsCommand
 115   blocklist?: BlocklistCommand
 116   subscriptions?: SubscriptionsCommand
 117   live?: LiveCommand
 118   services?: ServicesCommand
 119   blacklist?: BlacklistCommand
 120   captions?: CaptionsCommand
 121   changeOwnership?: ChangeOwnershipCommand
 122   playlists?: PlaylistsCommand
 123   history?: HistoryCommand
 124   imports?: ImportsCommand
 125   channelSyncs?: ChannelSyncsCommand
 126   streamingPlaylists?: StreamingPlaylistsCommand
 127   channels?: ChannelsCommand
 128   comments?: CommentsCommand
 129   sql?: SQLCommand
 130   notifications?: NotificationsCommand
 131   servers?: ServersCommand
 132   login?: LoginCommand
 133   users?: UsersCommand
 134   objectStorage?: ObjectStorageCommand
 135   videoStudio?: VideoStudioCommand
 136   videos?: VideosCommand
 137   videoStats?: VideoStatsCommand
 138   views?: ViewsCommand
 139
 140   constructor (options: { serverNumber: number } | { url: string }) {
 141     if ((options as any).url) {
 142       this.setUrl((options as any).url)
 143     } else {
 144       this.setServerNumber((options as any).serverNumber)
 145     }
 146
 147     this.store = {
 148       client: {
 149         id: null,
 150         secret: null
 151       },
 152       user: {
 153         username: null,
 154         password: null
 155       }
 156     }
 157
 158     this.assignCommands()
 159   }
 160
 161   setServerNumber (serverNumber: number) {
 162     this.serverNumber = serverNumber
 163
 164     this.parallel = parallelTests()
 165
 166     this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber
 167     this.rtmpPort = this.parallel ? this.randomRTMP() : 1936
 168     this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937
 169     this.port = 9000 + this.internalServerNumber
 170
 171     this.url = `http://localhost:${this.port}`
 172     this.host = `localhost:${this.port}`
 173     this.hostname = 'localhost'
 174   }
 175
 176   setUrl (url: string) {
 177     const parsed = new URL(url)
 178
 179     this.url = url
 180     this.host = parsed.host
 181     this.hostname = parsed.hostname
 182     this.port = parseInt(parsed.port)
 183   }
 184
 185   async flushAndRun (configOverride?: Object, options: RunServerOptions = {}) {
 186     await ServersCommand.flushTests(this.internalServerNumber)
 187
 188     return this.run(configOverride, options)
 189   }
 190
 191   async run (configOverrideArg?: any, options: RunServerOptions = {}) {
 192     // These actions are async so we need to be sure that they have both been done
 193     const serverRunString = {
 194       'HTTP server listening': false
 195     }
 196     const key = 'Database peertube_test' + this.internalServerNumber + ' is ready'
 197     serverRunString[key] = false
 198
 199     const regexps = {
 200       client_id: 'Client id: (.+)',
 201       client_secret: 'Client secret: (.+)',
 202       user_username: 'Username: (.+)',
 203       user_password: 'User password: (.+)'
 204     }
 205
 206     await this.assignCustomConfigFile()
 207
 208     const configOverride = this.buildConfigOverride()
 209
 210     if (configOverrideArg !== undefined) {
 211       Object.assign(configOverride, configOverrideArg)
 212     }
 213
 214     // Share the environment
 215     const env = Object.create(process.env)
 216     env['NODE_ENV'] = 'test'
 217     env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
 218     env['NODE_CONFIG'] = JSON.stringify(configOverride)
 219
 220     if (options.env) {
 221       Object.assign(env, options.env)
 222     }
 223
 224     const execArgv = options.nodeArgs || []
 225     // FIXME: too slow :/
 226     // execArgv.push('--enable-source-maps')
 227
 228     const forkOptions = {
 229       silent: true,
 230       env,
 231       detached: true,
 232       execArgv
 233     }
 234
 235     const peertubeArgs = options.peertubeArgs || []
 236
 237     return new Promise<void>((res, rej) => {
 238       const self = this
 239       let aggregatedLogs = ''
 240
 241       this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions)
 242
 243       const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs))
 244       const onParentExit = () => {
 245         if (!this.app || !this.app.pid) return
 246
 247         try {
 248           process.kill(self.app.pid)
 249         } catch { /* empty */ }
 250       }
 251
 252       this.app.on('exit', onPeerTubeExit)
 253       process.on('exit', onParentExit)
 254
 255       this.app.stdout.on('data', function onStdout (data) {
 256         let dontContinue = false
 257
 258         const log: string = data.toString()
 259         aggregatedLogs += log
 260
 261         // Capture things if we want to
 262         for (const key of Object.keys(regexps)) {
 263           const regexp = regexps[key]
 264           const matches = log.match(regexp)
 265           if (matches !== null) {
 266             if (key === 'client_id') self.store.client.id = matches[1]
 267             else if (key === 'client_secret') self.store.client.secret = matches[1]
 268             else if (key === 'user_username') self.store.user.username = matches[1]
 269             else if (key === 'user_password') self.store.user.password = matches[1]
 270           }
 271         }
 272
 273         // Check if all required sentences are here
 274         for (const key of Object.keys(serverRunString)) {
 275           if (log.includes(key)) serverRunString[key] = true
 276           if (serverRunString[key] === false) dontContinue = true
 277         }
 278
 279         // If no, there is maybe one thing not already initialized (client/user credentials generation...)
 280         if (dontContinue === true) return
 281
 282         if (options.hideLogs === false) {
 283           console.log(log)
 284         } else {
 285           process.removeListener('exit', onParentExit)
 286           self.app.stdout.removeListener('data', onStdout)
 287           self.app.removeListener('exit', onPeerTubeExit)
 288         }
 289
 290         res()
 291       })
 292     })
 293   }
 294
 295   async kill () {
 296     if (!this.app) return
 297
 298     await this.sql.cleanup()
 299
 300     process.kill(-this.app.pid)
 301
 302     this.app = null
 303   }
 304
 305   private randomServer () {
 306     const low = 10
 307     const high = 10000
 308
 309     return randomInt(low, high)
 310   }
 311
 312   private randomRTMP () {
 313     const low = 1900
 314     const high = 2100
 315
 316     return randomInt(low, high)
 317   }
 318
 319   private async assignCustomConfigFile () {
 320     if (this.internalServerNumber === this.serverNumber) return
 321
 322     const basePath = join(root(), 'config')
 323
 324     const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`)
 325     await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile)
 326
 327     this.customConfigFile = tmpConfigFile
 328   }
 329
 330   private buildConfigOverride () {
 331     if (!this.parallel) return {}
 332
 333     return {
 334       listen: {
 335         port: this.port
 336       },
 337       webserver: {
 338         port: this.port
 339       },
 340       database: {
 341         suffix: '_test' + this.internalServerNumber
 342       },
 343       storage: {
 344         tmp: `test${this.internalServerNumber}/tmp/`,
 345         bin: `test${this.internalServerNumber}/bin/`,
 346         avatars: `test${this.internalServerNumber}/avatars/`,
 347         videos: `test${this.internalServerNumber}/videos/`,
 348         streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`,
 349         redundancy: `test${this.internalServerNumber}/redundancy/`,
 350         logs: `test${this.internalServerNumber}/logs/`,
 351         previews: `test${this.internalServerNumber}/previews/`,
 352         thumbnails: `test${this.internalServerNumber}/thumbnails/`,
 353         torrents: `test${this.internalServerNumber}/torrents/`,
 354         captions: `test${this.internalServerNumber}/captions/`,
 355         cache: `test${this.internalServerNumber}/cache/`,
 356         plugins: `test${this.internalServerNumber}/plugins/`
 357       },
 358       admin: {
 359         email: `admin${this.internalServerNumber}@example.com`
 360       },
 361       live: {
 362         rtmp: {
 363           port: this.rtmpPort
 364         }
 365       }
 366     }
 367   }
 368
 369   private assignCommands () {
 370     this.bulk = new BulkCommand(this)
 371     this.cli = new CLICommand(this)
 372     this.customPage = new CustomPagesCommand(this)
 373     this.feed = new FeedCommand(this)
 374     this.logs = new LogsCommand(this)
 375     this.abuses = new AbusesCommand(this)
 376     this.overviews = new OverviewsCommand(this)
 377     this.search = new SearchCommand(this)
 378     this.contactForm = new ContactFormCommand(this)
 379     this.debug = new DebugCommand(this)
 380     this.follows = new FollowsCommand(this)
 381     this.jobs = new JobsCommand(this)
 382     this.metrics = new MetricsCommand(this)
 383     this.plugins = new PluginsCommand(this)
 384     this.redundancy = new RedundancyCommand(this)
 385     this.stats = new StatsCommand(this)
 386     this.config = new ConfigCommand(this)
 387     this.socketIO = new SocketIOCommand(this)
 388     this.accounts = new AccountsCommand(this)
 389     this.blocklist = new BlocklistCommand(this)
 390     this.subscriptions = new SubscriptionsCommand(this)
 391     this.live = new LiveCommand(this)
 392     this.services = new ServicesCommand(this)
 393     this.blacklist = new BlacklistCommand(this)
 394     this.captions = new CaptionsCommand(this)
 395     this.changeOwnership = new ChangeOwnershipCommand(this)
 396     this.playlists = new PlaylistsCommand(this)
 397     this.history = new HistoryCommand(this)
 398     this.imports = new ImportsCommand(this)
 399     this.channelSyncs = new ChannelSyncsCommand(this)
 400     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
 401     this.channels = new ChannelsCommand(this)
 402     this.comments = new CommentsCommand(this)
 403     this.sql = new SQLCommand(this)
 404     this.notifications = new NotificationsCommand(this)
 405     this.servers = new ServersCommand(this)
 406     this.login = new LoginCommand(this)
 407     this.users = new UsersCommand(this)
 408     this.videos = new VideosCommand(this)
 409     this.objectStorage = new ObjectStorageCommand(this)
 410     this.videoStudio = new VideoStudioCommand(this)
 411     this.videoStats = new VideoStatsCommand(this)
 412     this.views = new ViewsCommand(this)
 413   }
 414 }