shared/server-commands/server/server.ts

   1 import { ChildProcess, fork } from 'child_process'
   2 import { copy } from 'fs-extra'
   3 import { join } from 'path'
   4 import { parallelTests, randomInt, root } from '@shared/core-utils'
   5 import { Video, VideoChannel, VideoCreateResult, VideoDetails } from '@shared/models'
   6 import { BulkCommand } from '../bulk'
   7 import { CLICommand } from '../cli'
   8 import { CustomPagesCommand } from '../custom-pages'
   9 import { FeedCommand } from '../feeds'
  10 import { LogsCommand } from '../logs'
  11 import { SQLCommand } from '../miscs'
  12 import { AbusesCommand } from '../moderation'
  13 import { OverviewsCommand } from '../overviews'
  14 import { SearchCommand } from '../search'
  15 import { SocketIOCommand } from '../socket'
  16 import { AccountsCommand, BlocklistCommand, LoginCommand, NotificationsCommand, SubscriptionsCommand, UsersCommand } from '../users'
  17 import {
  18   BlacklistCommand,
  19   CaptionsCommand,
  20   ChangeOwnershipCommand,
  21   ChannelsCommand,
  22   HistoryCommand,
  23   ImportsCommand,
  24   LiveCommand,
  25   PlaylistsCommand,
  26   ServicesCommand,
  27   StreamingPlaylistsCommand,
  28   VideosCommand
  29 } from '../videos'
  30 import { CommentsCommand } from '../videos/comments-command'
  31 import { ConfigCommand } from './config-command'
  32 import { ContactFormCommand } from './contact-form-command'
  33 import { DebugCommand } from './debug-command'
  34 import { FollowsCommand } from './follows-command'
  35 import { JobsCommand } from './jobs-command'
  36 import { ObjectStorageCommand } from './object-storage-command'
  37 import { PluginsCommand } from './plugins-command'
  38 import { RedundancyCommand } from './redundancy-command'
  39 import { ServersCommand } from './servers-command'
  40 import { StatsCommand } from './stats-command'
  41
  42 export type RunServerOptions = {
  43   hideLogs?: boolean
  44   nodeArgs?: string[]
  45   peertubeArgs?: string[]
  46   env?: { [ id: string ]: string }
  47 }
  48
  49 export class PeerTubeServer {
  50   app?: ChildProcess
  51
  52   url: string
  53   host?: string
  54   hostname?: string
  55   port?: number
  56
  57   rtmpPort?: number
  58   rtmpsPort?: number
  59
  60   parallel?: boolean
  61   internalServerNumber: number
  62
  63   serverNumber?: number
  64   customConfigFile?: string
  65
  66   store?: {
  67     client?: {
  68       id?: string
  69       secret?: string
  70     }
  71
  72     user?: {
  73       username: string
  74       password: string
  75       email?: string
  76     }
  77
  78     channel?: VideoChannel
  79
  80     video?: Video
  81     videoCreated?: VideoCreateResult
  82     videoDetails?: VideoDetails
  83
  84     videos?: { id: number, uuid: string }[]
  85   }
  86
  87   accessToken?: string
  88   refreshToken?: string
  89
  90   bulk?: BulkCommand
  91   cli?: CLICommand
  92   customPage?: CustomPagesCommand
  93   feed?: FeedCommand
  94   logs?: LogsCommand
  95   abuses?: AbusesCommand
  96   overviews?: OverviewsCommand
  97   search?: SearchCommand
  98   contactForm?: ContactFormCommand
  99   debug?: DebugCommand
 100   follows?: FollowsCommand
 101   jobs?: JobsCommand
 102   plugins?: PluginsCommand
 103   redundancy?: RedundancyCommand
 104   stats?: StatsCommand
 105   config?: ConfigCommand
 106   socketIO?: SocketIOCommand
 107   accounts?: AccountsCommand
 108   blocklist?: BlocklistCommand
 109   subscriptions?: SubscriptionsCommand
 110   live?: LiveCommand
 111   services?: ServicesCommand
 112   blacklist?: BlacklistCommand
 113   captions?: CaptionsCommand
 114   changeOwnership?: ChangeOwnershipCommand
 115   playlists?: PlaylistsCommand
 116   history?: HistoryCommand
 117   imports?: ImportsCommand
 118   streamingPlaylists?: StreamingPlaylistsCommand
 119   channels?: ChannelsCommand
 120   comments?: CommentsCommand
 121   sql?: SQLCommand
 122   notifications?: NotificationsCommand
 123   servers?: ServersCommand
 124   login?: LoginCommand
 125   users?: UsersCommand
 126   objectStorage?: ObjectStorageCommand
 127   videos?: VideosCommand
 128
 129   constructor (options: { serverNumber: number } | { url: string }) {
 130     if ((options as any).url) {
 131       this.setUrl((options as any).url)
 132     } else {
 133       this.setServerNumber((options as any).serverNumber)
 134     }
 135
 136     this.store = {
 137       client: {
 138         id: null,
 139         secret: null
 140       },
 141       user: {
 142         username: null,
 143         password: null
 144       }
 145     }
 146
 147     this.assignCommands()
 148   }
 149
 150   setServerNumber (serverNumber: number) {
 151     this.serverNumber = serverNumber
 152
 153     this.parallel = parallelTests()
 154
 155     this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber
 156     this.rtmpPort = this.parallel ? this.randomRTMP() : 1936
 157     this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937
 158     this.port = 9000 + this.internalServerNumber
 159
 160     this.url = `http://localhost:${this.port}`
 161     this.host = `localhost:${this.port}`
 162     this.hostname = 'localhost'
 163   }
 164
 165   setUrl (url: string) {
 166     const parsed = new URL(url)
 167
 168     this.url = url
 169     this.host = parsed.host
 170     this.hostname = parsed.hostname
 171     this.port = parseInt(parsed.port)
 172   }
 173
 174   async flushAndRun (configOverride?: Object, options: RunServerOptions = {}) {
 175     await ServersCommand.flushTests(this.internalServerNumber)
 176
 177     return this.run(configOverride, options)
 178   }
 179
 180   async run (configOverrideArg?: any, options: RunServerOptions = {}) {
 181     // These actions are async so we need to be sure that they have both been done
 182     const serverRunString = {
 183       'HTTP server listening': false
 184     }
 185     const key = 'Database peertube_test' + this.internalServerNumber + ' is ready'
 186     serverRunString[key] = false
 187
 188     const regexps = {
 189       client_id: 'Client id: (.+)',
 190       client_secret: 'Client secret: (.+)',
 191       user_username: 'Username: (.+)',
 192       user_password: 'User password: (.+)'
 193     }
 194
 195     await this.assignCustomConfigFile()
 196
 197     const configOverride = this.buildConfigOverride()
 198
 199     if (configOverrideArg !== undefined) {
 200       Object.assign(configOverride, configOverrideArg)
 201     }
 202
 203     // Share the environment
 204     const env = Object.create(process.env)
 205     env['NODE_ENV'] = 'test'
 206     env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
 207     env['NODE_CONFIG'] = JSON.stringify(configOverride)
 208
 209     if (options.env) {
 210       Object.assign(env, options.env)
 211     }
 212
 213     const execArgv = options.nodeArgs || []
 214     execArgv.push('--enable-source-maps')
 215
 216     const forkOptions = {
 217       silent: true,
 218       env,
 219       detached: true,
 220       execArgv
 221     }
 222
 223     const peertubeArgs = options.peertubeArgs || []
 224
 225     return new Promise<void>((res, rej) => {
 226       const self = this
 227       let aggregatedLogs = ''
 228
 229       this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions)
 230
 231       const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs))
 232       const onParentExit = () => {
 233         if (!this.app || !this.app.pid) return
 234
 235         try {
 236           process.kill(self.app.pid)
 237         } catch { /* empty */ }
 238       }
 239
 240       this.app.on('exit', onPeerTubeExit)
 241       process.on('exit', onParentExit)
 242
 243       this.app.stdout.on('data', function onStdout (data) {
 244         let dontContinue = false
 245
 246         const log: string = data.toString()
 247         aggregatedLogs += log
 248
 249         // Capture things if we want to
 250         for (const key of Object.keys(regexps)) {
 251           const regexp = regexps[key]
 252           const matches = log.match(regexp)
 253           if (matches !== null) {
 254             if (key === 'client_id') self.store.client.id = matches[1]
 255             else if (key === 'client_secret') self.store.client.secret = matches[1]
 256             else if (key === 'user_username') self.store.user.username = matches[1]
 257             else if (key === 'user_password') self.store.user.password = matches[1]
 258           }
 259         }
 260
 261         // Check if all required sentences are here
 262         for (const key of Object.keys(serverRunString)) {
 263           if (log.includes(key)) serverRunString[key] = true
 264           if (serverRunString[key] === false) dontContinue = true
 265         }
 266
 267         // If no, there is maybe one thing not already initialized (client/user credentials generation...)
 268         if (dontContinue === true) return
 269
 270         if (options.hideLogs === false) {
 271           console.log(log)
 272         } else {
 273           process.removeListener('exit', onParentExit)
 274           self.app.stdout.removeListener('data', onStdout)
 275           self.app.removeListener('exit', onPeerTubeExit)
 276         }
 277
 278         res()
 279       })
 280     })
 281   }
 282
 283   async kill () {
 284     if (!this.app) return
 285
 286     await this.sql.cleanup()
 287
 288     process.kill(-this.app.pid)
 289
 290     this.app = null
 291   }
 292
 293   private randomServer () {
 294     const low = 10
 295     const high = 10000
 296
 297     return randomInt(low, high)
 298   }
 299
 300   private randomRTMP () {
 301     const low = 1900
 302     const high = 2100
 303
 304     return randomInt(low, high)
 305   }
 306
 307   private async assignCustomConfigFile () {
 308     if (this.internalServerNumber === this.serverNumber) return
 309
 310     const basePath = join(root(), 'config')
 311
 312     const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`)
 313     await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile)
 314
 315     this.customConfigFile = tmpConfigFile
 316   }
 317
 318   private buildConfigOverride () {
 319     if (!this.parallel) return {}
 320
 321     return {
 322       listen: {
 323         port: this.port
 324       },
 325       webserver: {
 326         port: this.port
 327       },
 328       database: {
 329         suffix: '_test' + this.internalServerNumber
 330       },
 331       storage: {
 332         tmp: `test${this.internalServerNumber}/tmp/`,
 333         bin: `test${this.internalServerNumber}/bin/`,
 334         avatars: `test${this.internalServerNumber}/avatars/`,
 335         videos: `test${this.internalServerNumber}/videos/`,
 336         streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`,
 337         redundancy: `test${this.internalServerNumber}/redundancy/`,
 338         logs: `test${this.internalServerNumber}/logs/`,
 339         previews: `test${this.internalServerNumber}/previews/`,
 340         thumbnails: `test${this.internalServerNumber}/thumbnails/`,
 341         torrents: `test${this.internalServerNumber}/torrents/`,
 342         captions: `test${this.internalServerNumber}/captions/`,
 343         cache: `test${this.internalServerNumber}/cache/`,
 344         plugins: `test${this.internalServerNumber}/plugins/`
 345       },
 346       admin: {
 347         email: `admin${this.internalServerNumber}@example.com`
 348       },
 349       live: {
 350         rtmp: {
 351           port: this.rtmpPort
 352         }
 353       }
 354     }
 355   }
 356
 357   private assignCommands () {
 358     this.bulk = new BulkCommand(this)
 359     this.cli = new CLICommand(this)
 360     this.customPage = new CustomPagesCommand(this)
 361     this.feed = new FeedCommand(this)
 362     this.logs = new LogsCommand(this)
 363     this.abuses = new AbusesCommand(this)
 364     this.overviews = new OverviewsCommand(this)
 365     this.search = new SearchCommand(this)
 366     this.contactForm = new ContactFormCommand(this)
 367     this.debug = new DebugCommand(this)
 368     this.follows = new FollowsCommand(this)
 369     this.jobs = new JobsCommand(this)
 370     this.plugins = new PluginsCommand(this)
 371     this.redundancy = new RedundancyCommand(this)
 372     this.stats = new StatsCommand(this)
 373     this.config = new ConfigCommand(this)
 374     this.socketIO = new SocketIOCommand(this)
 375     this.accounts = new AccountsCommand(this)
 376     this.blocklist = new BlocklistCommand(this)
 377     this.subscriptions = new SubscriptionsCommand(this)
 378     this.live = new LiveCommand(this)
 379     this.services = new ServicesCommand(this)
 380     this.blacklist = new BlacklistCommand(this)
 381     this.captions = new CaptionsCommand(this)
 382     this.changeOwnership = new ChangeOwnershipCommand(this)
 383     this.playlists = new PlaylistsCommand(this)
 384     this.history = new HistoryCommand(this)
 385     this.imports = new ImportsCommand(this)
 386     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
 387     this.channels = new ChannelsCommand(this)
 388     this.comments = new CommentsCommand(this)
 389     this.sql = new SQLCommand(this)
 390     this.notifications = new NotificationsCommand(this)
 391     this.servers = new ServersCommand(this)
 392     this.login = new LoginCommand(this)
 393     this.users = new UsersCommand(this)
 394     this.videos = new VideosCommand(this)
 395     this.objectStorage = new ObjectStorageCommand(this)
 396   }
 397 }