shared/extra-utils/server/server.ts

   1 import { ChildProcess, fork } from 'child_process'
   2 import { copy } from 'fs-extra'
   3 import { join } from 'path'
   4 import { root } from '@server/helpers/core-utils'
   5 import { randomInt } from '../../core-utils/miscs/miscs'
   6 import { VideoChannel } from '../../models/videos'
   7 import { BulkCommand } from '../bulk'
   8 import { CLICommand } from '../cli'
   9 import { CustomPagesCommand } from '../custom-pages'
  10 import { FeedCommand } from '../feeds'
  11 import { LogsCommand } from '../logs'
  12 import { parallelTests, SQLCommand } from '../miscs'
  13 import { AbusesCommand } from '../moderation'
  14 import { OverviewsCommand } from '../overviews'
  15 import { SearchCommand } from '../search'
  16 import { SocketIOCommand } from '../socket'
  17 import { AccountsCommand, BlocklistCommand, LoginCommand, NotificationsCommand, SubscriptionsCommand, UsersCommand } from '../users'
  18 import {
  19   BlacklistCommand,
  20   CaptionsCommand,
  21   ChangeOwnershipCommand,
  22   ChannelsCommand,
  23   HistoryCommand,
  24   ImportsCommand,
  25   LiveCommand,
  26   PlaylistsCommand,
  27   ServicesCommand,
  28   StreamingPlaylistsCommand,
  29   VideosCommand
  30 } from '../videos'
  31 import { CommentsCommand } from '../videos/comments-command'
  32 import { ConfigCommand } from './config-command'
  33 import { ContactFormCommand } from './contact-form-command'
  34 import { DebugCommand } from './debug-command'
  35 import { FollowsCommand } from './follows-command'
  36 import { JobsCommand } from './jobs-command'
  37 import { PluginsCommand } from './plugins-command'
  38 import { RedundancyCommand } from './redundancy-command'
  39 import { ServersCommand } from './servers-command'
  40 import { StatsCommand } from './stats-command'
  41
  42 export type RunServerOptions = {
  43   hideLogs?: boolean
  44   execArgv?: string[]
  45 }
  46
  47 export class PeerTubeServer {
  48   app?: ChildProcess
  49
  50   url: string
  51   host?: string
  52   hostname?: string
  53   port?: number
  54
  55   rtmpPort?: number
  56
  57   parallel?: boolean
  58   internalServerNumber: number
  59
  60   serverNumber?: number
  61   customConfigFile?: string
  62
  63   store?: {
  64     client?: {
  65       id?: string
  66       secret?: string
  67     }
  68
  69     user?: {
  70       username: string
  71       password: string
  72       email?: string
  73     }
  74
  75     channel?: VideoChannel
  76
  77     video?: {
  78       id: number
  79       uuid: string
  80       shortUUID: string
  81       name?: string
  82       url?: string
  83
  84       account?: {
  85         name: string
  86       }
  87
  88       embedPath?: string
  89     }
  90
  91     videos?: { id: number, uuid: string }[]
  92   }
  93
  94   accessToken?: string
  95   refreshToken?: string
  96
  97   bulk?: BulkCommand
  98   cli?: CLICommand
  99   customPage?: CustomPagesCommand
 100   feed?: FeedCommand
 101   logs?: LogsCommand
 102   abuses?: AbusesCommand
 103   overviews?: OverviewsCommand
 104   search?: SearchCommand
 105   contactForm?: ContactFormCommand
 106   debug?: DebugCommand
 107   follows?: FollowsCommand
 108   jobs?: JobsCommand
 109   plugins?: PluginsCommand
 110   redundancy?: RedundancyCommand
 111   stats?: StatsCommand
 112   config?: ConfigCommand
 113   socketIO?: SocketIOCommand
 114   accounts?: AccountsCommand
 115   blocklist?: BlocklistCommand
 116   subscriptions?: SubscriptionsCommand
 117   live?: LiveCommand
 118   services?: ServicesCommand
 119   blacklist?: BlacklistCommand
 120   captions?: CaptionsCommand
 121   changeOwnership?: ChangeOwnershipCommand
 122   playlists?: PlaylistsCommand
 123   history?: HistoryCommand
 124   imports?: ImportsCommand
 125   streamingPlaylists?: StreamingPlaylistsCommand
 126   channels?: ChannelsCommand
 127   comments?: CommentsCommand
 128   sql?: SQLCommand
 129   notifications?: NotificationsCommand
 130   servers?: ServersCommand
 131   login?: LoginCommand
 132   users?: UsersCommand
 133   videos?: VideosCommand
 134
 135   constructor (options: { serverNumber: number } | { url: string }) {
 136     if ((options as any).url) {
 137       this.setUrl((options as any).url)
 138     } else {
 139       this.setServerNumber((options as any).serverNumber)
 140     }
 141
 142     this.store = {
 143       client: {
 144         id: null,
 145         secret: null
 146       },
 147       user: {
 148         username: null,
 149         password: null
 150       }
 151     }
 152
 153     this.assignCommands()
 154   }
 155
 156   setServerNumber (serverNumber: number) {
 157     this.serverNumber = serverNumber
 158
 159     this.parallel = parallelTests()
 160
 161     this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber
 162     this.rtmpPort = this.parallel ? this.randomRTMP() : 1936
 163     this.port = 9000 + this.internalServerNumber
 164
 165     this.url = `http://localhost:${this.port}`
 166     this.host = `localhost:${this.port}`
 167     this.hostname = 'localhost'
 168   }
 169
 170   setUrl (url: string) {
 171     const parsed = new URL(url)
 172
 173     this.url = url
 174     this.host = parsed.host
 175     this.hostname = parsed.hostname
 176     this.port = parseInt(parsed.port)
 177   }
 178
 179   async flushAndRun (configOverride?: Object, args = [], options: RunServerOptions = {}) {
 180     await ServersCommand.flushTests(this.internalServerNumber)
 181
 182     return this.run(configOverride, args, options)
 183   }
 184
 185   async run (configOverrideArg?: any, args = [], options: RunServerOptions = {}) {
 186     // These actions are async so we need to be sure that they have both been done
 187     const serverRunString = {
 188       'HTTP server listening': false
 189     }
 190     const key = 'Database peertube_test' + this.internalServerNumber + ' is ready'
 191     serverRunString[key] = false
 192
 193     const regexps = {
 194       client_id: 'Client id: (.+)',
 195       client_secret: 'Client secret: (.+)',
 196       user_username: 'Username: (.+)',
 197       user_password: 'User password: (.+)'
 198     }
 199
 200     await this.assignCustomConfigFile()
 201
 202     const configOverride = this.buildConfigOverride()
 203
 204     if (configOverrideArg !== undefined) {
 205       Object.assign(configOverride, configOverrideArg)
 206     }
 207
 208     // Share the environment
 209     const env = Object.create(process.env)
 210     env['NODE_ENV'] = 'test'
 211     env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
 212     env['NODE_CONFIG'] = JSON.stringify(configOverride)
 213
 214     const forkOptions = {
 215       silent: true,
 216       env,
 217       detached: true,
 218       execArgv: options.execArgv || []
 219     }
 220
 221     return new Promise<void>(res => {
 222       this.app = fork(join(root(), 'dist', 'server.js'), args, forkOptions)
 223       this.app.stdout.on('data', function onStdout (data) {
 224         let dontContinue = false
 225
 226         // Capture things if we want to
 227         for (const key of Object.keys(regexps)) {
 228           const regexp = regexps[key]
 229           const matches = data.toString().match(regexp)
 230           if (matches !== null) {
 231             if (key === 'client_id') this.store.client.id = matches[1]
 232             else if (key === 'client_secret') this.store.client.secret = matches[1]
 233             else if (key === 'user_username') this.store.user.username = matches[1]
 234             else if (key === 'user_password') this.store.user.password = matches[1]
 235           }
 236         }
 237
 238         // Check if all required sentences are here
 239         for (const key of Object.keys(serverRunString)) {
 240           if (data.toString().indexOf(key) !== -1) serverRunString[key] = true
 241           if (serverRunString[key] === false) dontContinue = true
 242         }
 243
 244         // If no, there is maybe one thing not already initialized (client/user credentials generation...)
 245         if (dontContinue === true) return
 246
 247         if (options.hideLogs === false) {
 248           console.log(data.toString())
 249         } else {
 250           this.app.stdout.removeListener('data', onStdout)
 251         }
 252
 253         process.on('exit', () => {
 254           try {
 255             process.kill(this.server.app.pid)
 256           } catch { /* empty */ }
 257         })
 258
 259         res()
 260       })
 261     })
 262   }
 263
 264   async kill () {
 265     if (!this.app) return
 266
 267     await this.sql.cleanup()
 268
 269     process.kill(-this.app.pid)
 270
 271     this.app = null
 272   }
 273
 274   private randomServer () {
 275     const low = 10
 276     const high = 10000
 277
 278     return randomInt(low, high)
 279   }
 280
 281   private randomRTMP () {
 282     const low = 1900
 283     const high = 2100
 284
 285     return randomInt(low, high)
 286   }
 287
 288   private async assignCustomConfigFile () {
 289     if (this.internalServerNumber === this.serverNumber) return
 290
 291     const basePath = join(root(), 'config')
 292
 293     const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`)
 294     await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile)
 295
 296     this.customConfigFile = tmpConfigFile
 297   }
 298
 299   private buildConfigOverride () {
 300     if (!this.parallel) return {}
 301
 302     return {
 303       listen: {
 304         port: this.port
 305       },
 306       webserver: {
 307         port: this.port
 308       },
 309       database: {
 310         suffix: '_test' + this.internalServerNumber
 311       },
 312       storage: {
 313         tmp: `test${this.internalServerNumber}/tmp/`,
 314         avatars: `test${this.internalServerNumber}/avatars/`,
 315         videos: `test${this.internalServerNumber}/videos/`,
 316         streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`,
 317         redundancy: `test${this.internalServerNumber}/redundancy/`,
 318         logs: `test${this.internalServerNumber}/logs/`,
 319         previews: `test${this.internalServerNumber}/previews/`,
 320         thumbnails: `test${this.internalServerNumber}/thumbnails/`,
 321         torrents: `test${this.internalServerNumber}/torrents/`,
 322         captions: `test${this.internalServerNumber}/captions/`,
 323         cache: `test${this.internalServerNumber}/cache/`,
 324         plugins: `test${this.internalServerNumber}/plugins/`
 325       },
 326       admin: {
 327         email: `admin${this.internalServerNumber}@example.com`
 328       },
 329       live: {
 330         rtmp: {
 331           port: this.rtmpPort
 332         }
 333       }
 334     }
 335   }
 336
 337   private assignCommands () {
 338     this.bulk = new BulkCommand(this)
 339     this.cli = new CLICommand(this)
 340     this.customPage = new CustomPagesCommand(this)
 341     this.feed = new FeedCommand(this)
 342     this.logs = new LogsCommand(this)
 343     this.abuses = new AbusesCommand(this)
 344     this.overviews = new OverviewsCommand(this)
 345     this.search = new SearchCommand(this)
 346     this.contactForm = new ContactFormCommand(this)
 347     this.debug = new DebugCommand(this)
 348     this.follows = new FollowsCommand(this)
 349     this.jobs = new JobsCommand(this)
 350     this.plugins = new PluginsCommand(this)
 351     this.redundancy = new RedundancyCommand(this)
 352     this.stats = new StatsCommand(this)
 353     this.config = new ConfigCommand(this)
 354     this.socketIO = new SocketIOCommand(this)
 355     this.accounts = new AccountsCommand(this)
 356     this.blocklist = new BlocklistCommand(this)
 357     this.subscriptions = new SubscriptionsCommand(this)
 358     this.live = new LiveCommand(this)
 359     this.services = new ServicesCommand(this)
 360     this.blacklist = new BlacklistCommand(this)
 361     this.captions = new CaptionsCommand(this)
 362     this.changeOwnership = new ChangeOwnershipCommand(this)
 363     this.playlists = new PlaylistsCommand(this)
 364     this.history = new HistoryCommand(this)
 365     this.imports = new ImportsCommand(this)
 366     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
 367     this.channels = new ChannelsCommand(this)
 368     this.comments = new CommentsCommand(this)
 369     this.sql = new SQLCommand(this)
 370     this.notifications = new NotificationsCommand(this)
 371     this.servers = new ServersCommand(this)
 372     this.login = new LoginCommand(this)
 373     this.users = new UsersCommand(this)
 374     this.videos = new VideosCommand(this)
 375   }
 376 }