shared/extra-utils/server/server.ts

   1 import { ChildProcess, fork } from 'child_process'
   2 import { copy } from 'fs-extra'
   3 import { join } from 'path'
   4 import { root } from '@server/helpers/core-utils'
   5 import { randomInt } from '@shared/core-utils'
   6 import { Video, VideoChannel, VideoCreateResult, VideoDetails } from '../../models/videos'
   7 import { BulkCommand } from '../bulk'
   8 import { CLICommand } from '../cli'
   9 import { CustomPagesCommand } from '../custom-pages'
  10 import { FeedCommand } from '../feeds'
  11 import { LogsCommand } from '../logs'
  12 import { parallelTests, SQLCommand } from '../miscs'
  13 import { AbusesCommand } from '../moderation'
  14 import { OverviewsCommand } from '../overviews'
  15 import { SearchCommand } from '../search'
  16 import { SocketIOCommand } from '../socket'
  17 import { AccountsCommand, BlocklistCommand, LoginCommand, NotificationsCommand, SubscriptionsCommand, UsersCommand } from '../users'
  18 import {
  19   BlacklistCommand,
  20   CaptionsCommand,
  21   ChangeOwnershipCommand,
  22   ChannelsCommand,
  23   HistoryCommand,
  24   ImportsCommand,
  25   LiveCommand,
  26   PlaylistsCommand,
  27   ServicesCommand,
  28   StreamingPlaylistsCommand,
  29   VideosCommand
  30 } from '../videos'
  31 import { CommentsCommand } from '../videos/comments-command'
  32 import { ConfigCommand } from './config-command'
  33 import { ContactFormCommand } from './contact-form-command'
  34 import { DebugCommand } from './debug-command'
  35 import { FollowsCommand } from './follows-command'
  36 import { JobsCommand } from './jobs-command'
  37 import { PluginsCommand } from './plugins-command'
  38 import { RedundancyCommand } from './redundancy-command'
  39 import { ServersCommand } from './servers-command'
  40 import { StatsCommand } from './stats-command'
  41
  42 export type RunServerOptions = {
  43   hideLogs?: boolean
  44   nodeArgs?: string[]
  45   peertubeArgs?: string[]
  46 }
  47
  48 export class PeerTubeServer {
  49   app?: ChildProcess
  50
  51   url: string
  52   host?: string
  53   hostname?: string
  54   port?: number
  55
  56   rtmpPort?: number
  57
  58   parallel?: boolean
  59   internalServerNumber: number
  60
  61   serverNumber?: number
  62   customConfigFile?: string
  63
  64   store?: {
  65     client?: {
  66       id?: string
  67       secret?: string
  68     }
  69
  70     user?: {
  71       username: string
  72       password: string
  73       email?: string
  74     }
  75
  76     channel?: VideoChannel
  77
  78     video?: Video
  79     videoCreated?: VideoCreateResult
  80     videoDetails?: VideoDetails
  81
  82     videos?: { id: number, uuid: string }[]
  83   }
  84
  85   accessToken?: string
  86   refreshToken?: string
  87
  88   bulk?: BulkCommand
  89   cli?: CLICommand
  90   customPage?: CustomPagesCommand
  91   feed?: FeedCommand
  92   logs?: LogsCommand
  93   abuses?: AbusesCommand
  94   overviews?: OverviewsCommand
  95   search?: SearchCommand
  96   contactForm?: ContactFormCommand
  97   debug?: DebugCommand
  98   follows?: FollowsCommand
  99   jobs?: JobsCommand
 100   plugins?: PluginsCommand
 101   redundancy?: RedundancyCommand
 102   stats?: StatsCommand
 103   config?: ConfigCommand
 104   socketIO?: SocketIOCommand
 105   accounts?: AccountsCommand
 106   blocklist?: BlocklistCommand
 107   subscriptions?: SubscriptionsCommand
 108   live?: LiveCommand
 109   services?: ServicesCommand
 110   blacklist?: BlacklistCommand
 111   captions?: CaptionsCommand
 112   changeOwnership?: ChangeOwnershipCommand
 113   playlists?: PlaylistsCommand
 114   history?: HistoryCommand
 115   imports?: ImportsCommand
 116   streamingPlaylists?: StreamingPlaylistsCommand
 117   channels?: ChannelsCommand
 118   comments?: CommentsCommand
 119   sql?: SQLCommand
 120   notifications?: NotificationsCommand
 121   servers?: ServersCommand
 122   login?: LoginCommand
 123   users?: UsersCommand
 124   videos?: VideosCommand
 125
 126   constructor (options: { serverNumber: number } | { url: string }) {
 127     if ((options as any).url) {
 128       this.setUrl((options as any).url)
 129     } else {
 130       this.setServerNumber((options as any).serverNumber)
 131     }
 132
 133     this.store = {
 134       client: {
 135         id: null,
 136         secret: null
 137       },
 138       user: {
 139         username: null,
 140         password: null
 141       }
 142     }
 143
 144     this.assignCommands()
 145   }
 146
 147   setServerNumber (serverNumber: number) {
 148     this.serverNumber = serverNumber
 149
 150     this.parallel = parallelTests()
 151
 152     this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber
 153     this.rtmpPort = this.parallel ? this.randomRTMP() : 1936
 154     this.port = 9000 + this.internalServerNumber
 155
 156     this.url = `http://localhost:${this.port}`
 157     this.host = `localhost:${this.port}`
 158     this.hostname = 'localhost'
 159   }
 160
 161   setUrl (url: string) {
 162     const parsed = new URL(url)
 163
 164     this.url = url
 165     this.host = parsed.host
 166     this.hostname = parsed.hostname
 167     this.port = parseInt(parsed.port)
 168   }
 169
 170   async flushAndRun (configOverride?: Object, options: RunServerOptions = {}) {
 171     await ServersCommand.flushTests(this.internalServerNumber)
 172
 173     return this.run(configOverride, options)
 174   }
 175
 176   async run (configOverrideArg?: any, options: RunServerOptions = {}) {
 177     // These actions are async so we need to be sure that they have both been done
 178     const serverRunString = {
 179       'HTTP server listening': false
 180     }
 181     const key = 'Database peertube_test' + this.internalServerNumber + ' is ready'
 182     serverRunString[key] = false
 183
 184     const regexps = {
 185       client_id: 'Client id: (.+)',
 186       client_secret: 'Client secret: (.+)',
 187       user_username: 'Username: (.+)',
 188       user_password: 'User password: (.+)'
 189     }
 190
 191     await this.assignCustomConfigFile()
 192
 193     const configOverride = this.buildConfigOverride()
 194
 195     if (configOverrideArg !== undefined) {
 196       Object.assign(configOverride, configOverrideArg)
 197     }
 198
 199     // Share the environment
 200     const env = Object.create(process.env)
 201     env['NODE_ENV'] = 'test'
 202     env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
 203     env['NODE_CONFIG'] = JSON.stringify(configOverride)
 204
 205     const forkOptions = {
 206       silent: true,
 207       env,
 208       detached: true,
 209       execArgv: options.nodeArgs || []
 210     }
 211
 212     return new Promise<void>(res => {
 213       const self = this
 214
 215       this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions)
 216       this.app.stdout.on('data', function onStdout (data) {
 217         let dontContinue = false
 218
 219         // Capture things if we want to
 220         for (const key of Object.keys(regexps)) {
 221           const regexp = regexps[key]
 222           const matches = data.toString().match(regexp)
 223           if (matches !== null) {
 224             if (key === 'client_id') self.store.client.id = matches[1]
 225             else if (key === 'client_secret') self.store.client.secret = matches[1]
 226             else if (key === 'user_username') self.store.user.username = matches[1]
 227             else if (key === 'user_password') self.store.user.password = matches[1]
 228           }
 229         }
 230
 231         // Check if all required sentences are here
 232         for (const key of Object.keys(serverRunString)) {
 233           if (data.toString().indexOf(key) !== -1) serverRunString[key] = true
 234           if (serverRunString[key] === false) dontContinue = true
 235         }
 236
 237         // If no, there is maybe one thing not already initialized (client/user credentials generation...)
 238         if (dontContinue === true) return
 239
 240         if (options.hideLogs === false) {
 241           console.log(data.toString())
 242         } else {
 243           self.app.stdout.removeListener('data', onStdout)
 244         }
 245
 246         process.on('exit', () => {
 247           try {
 248             process.kill(self.app.pid)
 249           } catch { /* empty */ }
 250         })
 251
 252         res()
 253       })
 254     })
 255   }
 256
 257   async kill () {
 258     if (!this.app) return
 259
 260     await this.sql.cleanup()
 261
 262     process.kill(-this.app.pid)
 263
 264     this.app = null
 265   }
 266
 267   private randomServer () {
 268     const low = 10
 269     const high = 10000
 270
 271     return randomInt(low, high)
 272   }
 273
 274   private randomRTMP () {
 275     const low = 1900
 276     const high = 2100
 277
 278     return randomInt(low, high)
 279   }
 280
 281   private async assignCustomConfigFile () {
 282     if (this.internalServerNumber === this.serverNumber) return
 283
 284     const basePath = join(root(), 'config')
 285
 286     const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`)
 287     await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile)
 288
 289     this.customConfigFile = tmpConfigFile
 290   }
 291
 292   private buildConfigOverride () {
 293     if (!this.parallel) return {}
 294
 295     return {
 296       listen: {
 297         port: this.port
 298       },
 299       webserver: {
 300         port: this.port
 301       },
 302       database: {
 303         suffix: '_test' + this.internalServerNumber
 304       },
 305       storage: {
 306         tmp: `test${this.internalServerNumber}/tmp/`,
 307         avatars: `test${this.internalServerNumber}/avatars/`,
 308         videos: `test${this.internalServerNumber}/videos/`,
 309         streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`,
 310         redundancy: `test${this.internalServerNumber}/redundancy/`,
 311         logs: `test${this.internalServerNumber}/logs/`,
 312         previews: `test${this.internalServerNumber}/previews/`,
 313         thumbnails: `test${this.internalServerNumber}/thumbnails/`,
 314         torrents: `test${this.internalServerNumber}/torrents/`,
 315         captions: `test${this.internalServerNumber}/captions/`,
 316         cache: `test${this.internalServerNumber}/cache/`,
 317         plugins: `test${this.internalServerNumber}/plugins/`
 318       },
 319       admin: {
 320         email: `admin${this.internalServerNumber}@example.com`
 321       },
 322       live: {
 323         rtmp: {
 324           port: this.rtmpPort
 325         }
 326       }
 327     }
 328   }
 329
 330   private assignCommands () {
 331     this.bulk = new BulkCommand(this)
 332     this.cli = new CLICommand(this)
 333     this.customPage = new CustomPagesCommand(this)
 334     this.feed = new FeedCommand(this)
 335     this.logs = new LogsCommand(this)
 336     this.abuses = new AbusesCommand(this)
 337     this.overviews = new OverviewsCommand(this)
 338     this.search = new SearchCommand(this)
 339     this.contactForm = new ContactFormCommand(this)
 340     this.debug = new DebugCommand(this)
 341     this.follows = new FollowsCommand(this)
 342     this.jobs = new JobsCommand(this)
 343     this.plugins = new PluginsCommand(this)
 344     this.redundancy = new RedundancyCommand(this)
 345     this.stats = new StatsCommand(this)
 346     this.config = new ConfigCommand(this)
 347     this.socketIO = new SocketIOCommand(this)
 348     this.accounts = new AccountsCommand(this)
 349     this.blocklist = new BlocklistCommand(this)
 350     this.subscriptions = new SubscriptionsCommand(this)
 351     this.live = new LiveCommand(this)
 352     this.services = new ServicesCommand(this)
 353     this.blacklist = new BlacklistCommand(this)
 354     this.captions = new CaptionsCommand(this)
 355     this.changeOwnership = new ChangeOwnershipCommand(this)
 356     this.playlists = new PlaylistsCommand(this)
 357     this.history = new HistoryCommand(this)
 358     this.imports = new ImportsCommand(this)
 359     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
 360     this.channels = new ChannelsCommand(this)
 361     this.comments = new CommentsCommand(this)
 362     this.sql = new SQLCommand(this)
 363     this.notifications = new NotificationsCommand(this)
 364     this.servers = new ServersCommand(this)
 365     this.login = new LoginCommand(this)
 366     this.users = new UsersCommand(this)
 367     this.videos = new VideosCommand(this)
 368   }
 369 }