Optimize rows parsing

1 files changed, 128 insertions, 96 deletions

diff --git a/server/models/video/sql/shared/video-model-builder.ts b/server/models/video/sql/shared/video-model-builder.ts
index 627ea6443..2a60dab04 100644
--- a/server/models/video/sql/shared/video-model-builder.ts
+++ b/server/models/video/sql/shared/video-model-builder.ts
@@ -1,4 +1,5 @@
-import { pick } from 'lodash'
+
+import { logger } from '@server/helpers/logger'
 import { AccountModel } from '@server/models/account/account'
 import { ActorModel } from '@server/models/actor/actor'
 import { ActorImageModel } from '@server/models/actor/actor-image'
@@ -15,7 +16,9 @@ import { VideoChannelModel } from '../../video-channel'
 import { VideoFileModel } from '../../video-file'
 import { VideoLiveModel } from '../../video-live'
 import { VideoStreamingPlaylistModel } from '../../video-streaming-playlist'
-import { VideoAttributes } from './video-attributes'
+import { VideoTables } from './video-tables'
+
+type SQLRow = { [id: string]: string | number }
 
 /**
  *
@@ -28,12 +31,12 @@ export class VideoModelBuilder {
   private videoStreamingPlaylistMemo: { [ id: number ]: VideoStreamingPlaylistModel }
   private videoFileMemo: { [ id: number ]: VideoFileModel }
 
-  private thumbnailsDone: Set<number>
-  private historyDone: Set<number>
-  private blacklistDone: Set<number>
-  private liveDone: Set<number>
-  private redundancyDone: Set<number>
-  private scheduleVideoUpdateDone: Set<number>
+  private thumbnailsDone: Set<any>
+  private historyDone: Set<any>
+  private blacklistDone: Set<any>
+  private liveDone: Set<any>
+  private redundancyDone: Set<any>
+  private scheduleVideoUpdateDone: Set<any>
 
   private trackersDone: Set<string>
   private tagsDone: Set<string>
@@ -44,12 +47,12 @@ export class VideoModelBuilder {
 
   constructor (
     readonly mode: 'get' | 'list',
-    readonly videoAttributes: VideoAttributes
+    readonly tables: VideoTables
   ) {
 
   }
 
-  buildVideosFromRows (rows: any[], rowsWebtorrentFiles?: any[], rowsStreamingPlaylist?: any[]) {
+  buildVideosFromRows (rows: SQLRow[], rowsWebTorrentFiles?: SQLRow[], rowsStreamingPlaylist?: SQLRow[]) {
     this.reinit()
 
     for (const row of rows) {
@@ -60,7 +63,7 @@ export class VideoModelBuilder {
       this.setUserHistory(row, videoModel)
       this.addThumbnail(row, videoModel)
 
-      if (!rowsWebtorrentFiles) {
+      if (!rowsWebTorrentFiles) {
         this.addWebTorrentFile(row, videoModel)
       }
 
@@ -75,30 +78,11 @@ export class VideoModelBuilder {
         this.setBlacklisted(row, videoModel)
         this.setScheduleVideoUpdate(row, videoModel)
         this.setLive(row, videoModel)
-
-        if (!rowsWebtorrentFiles && row.VideoFiles.id) {
-          this.addRedundancy(row.VideoFiles.RedundancyVideos, this.videoFileMemo[row.VideoFiles.id])
-        }
-
-        if (!rowsStreamingPlaylist && row.VideoStreamingPlaylists.id) {
-          this.addRedundancy(row.VideoStreamingPlaylists.RedundancyVideos, this.videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id])
-        }
       }
     }
 
-    for (const row of rowsWebtorrentFiles || []) {
-      const videoModel = this.videosMemo[row.id]
-      this.addWebTorrentFile(row, videoModel)
-      this.addRedundancy(row.VideoFiles.RedundancyVideos, this.videoFileMemo[row.VideoFiles.id])
-    }
-
-    for (const row of rowsStreamingPlaylist || []) {
-      const videoModel = this.videosMemo[row.id]
-
-      this.addStreamingPlaylist(row, videoModel)
-      this.addStreamingPlaylistFile(row)
-      this.addRedundancy(row.VideoStreamingPlaylists.RedundancyVideos, this.videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id])
-    }
+    this.grabSeparateWebTorrentFiles(rowsWebTorrentFiles)
+    this.grabSeparateStreamingPlaylistFiles(rowsStreamingPlaylist)
 
     return this.videos
   }
@@ -121,21 +105,45 @@ export class VideoModelBuilder {
     this.videos = []
   }
 
-  private buildVideo (row: any) {
+  private grabSeparateWebTorrentFiles (rowsWebTorrentFiles?: SQLRow[]) {
+    if (!rowsWebTorrentFiles) return
+
+    for (const row of rowsWebTorrentFiles) {
+      const videoModel = this.videosMemo[row.id]
+      this.addWebTorrentFile(row, videoModel)
+      this.addRedundancy(row, 'VideoFiles.RedundancyVideos', this.videoFileMemo[row['VideoFiles.id']])
+    }
+  }
+
+  private grabSeparateStreamingPlaylistFiles (rowsStreamingPlaylist?: SQLRow[]) {
+    if (!rowsStreamingPlaylist) return
+
+    for (const row of rowsStreamingPlaylist || []) {
+      const videoModel = this.videosMemo[row.id]
+
+      this.addStreamingPlaylist(row, videoModel)
+      this.addStreamingPlaylistFile(row)
+      this.addRedundancy(
+        row,
+        'VideoStreamingPlaylists.RedundancyVideos',
+        this.videoStreamingPlaylistMemo[row['VideoStreamingPlaylists.id']]
+      )
+    }
+  }
+
+  private buildVideo (row: SQLRow) {
     if (this.videosMemo[row.id]) return
 
     // Build Channel
-    const channel = row.VideoChannel
-    const channelModel = new VideoChannelModel(pick(channel, this.videoAttributes.getChannelAttributes()), this.buildOpts)
-    channelModel.Actor = this.buildActor(channel.Actor)
+    const channelModel = new VideoChannelModel(this.grab(row, this.tables.getChannelAttributes(), 'VideoChannel'), this.buildOpts)
+    channelModel.Actor = this.buildActor(row, 'VideoChannel')
 
-    const account = row.VideoChannel.Account
-    const accountModel = new AccountModel(pick(account, this.videoAttributes.getAccountAttributes()), this.buildOpts)
-    accountModel.Actor = this.buildActor(account.Actor)
+    const accountModel = new AccountModel(this.grab(row, this.tables.getAccountAttributes(), 'VideoChannel.Account'), this.buildOpts)
+    accountModel.Actor = this.buildActor(row, 'VideoChannel.Account')
 
     channelModel.Account = accountModel
 
-    const videoModel = new VideoModel(pick(row, this.videoAttributes.getVideoAttributes()), this.buildOpts)
+    const videoModel = new VideoModel(this.grab(row, this.tables.getVideoAttributes(), ''), this.buildOpts)
     videoModel.VideoChannel = channelModel
 
     this.videosMemo[row.id] = videoModel
@@ -151,143 +159,167 @@ export class VideoModelBuilder {
     this.videos.push(videoModel)
   }
 
-  private buildActor (rowActor: any) {
-    const avatarModel = rowActor.Avatar.id !== null
-      ? new ActorImageModel(pick(rowActor.Avatar, this.videoAttributes.getAvatarAttributes()), this.buildOpts)
+  private buildActor (row: SQLRow, prefix: string) {
+    const actorPrefix = `${prefix}.Actor`
+    const avatarPrefix = `${actorPrefix}.Avatar`
+    const serverPrefix = `${actorPrefix}.Server`
+
+    const avatarModel = row[`${avatarPrefix}.id`] !== null
+      ? new ActorImageModel(this.grab(row, this.tables.getAvatarAttributes(), avatarPrefix), this.buildOpts)
       : null
 
-    const serverModel = rowActor.Server.id !== null
-      ? new ServerModel(pick(rowActor.Server, this.videoAttributes.getServerAttributes()), this.buildOpts)
+    const serverModel = row[`${serverPrefix}.id`] !== null
+      ? new ServerModel(this.grab(row, this.tables.getServerAttributes(), serverPrefix), this.buildOpts)
       : null
 
-    const actorModel = new ActorModel(pick(rowActor, this.videoAttributes.getActorAttributes()), this.buildOpts)
+    const actorModel = new ActorModel(this.grab(row, this.tables.getActorAttributes(), actorPrefix), this.buildOpts)
     actorModel.Avatar = avatarModel
     actorModel.Server = serverModel
 
     return actorModel
   }
 
-  private setUserHistory (row: any, videoModel: VideoModel) {
-    if (!row.userVideoHistory?.id || this.historyDone.has(row.userVideoHistory.id)) return
+  private setUserHistory (row: SQLRow, videoModel: VideoModel) {
+    const id = row['userVideoHistory.id']
+    if (!id || this.historyDone.has(id)) return
 
-    const attributes = pick(row.userVideoHistory, this.videoAttributes.getUserHistoryAttributes())
+    const attributes = this.grab(row, this.tables.getUserHistoryAttributes(), 'userVideoHistory')
     const historyModel = new UserVideoHistoryModel(attributes, this.buildOpts)
     videoModel.UserVideoHistories.push(historyModel)
 
-    this.historyDone.add(row.userVideoHistory.id)
+    this.historyDone.add(id)
   }
 
-  private addThumbnail (row: any, videoModel: VideoModel) {
-    if (!row.Thumbnails?.id || this.thumbnailsDone.has(row.Thumbnails.id)) return
+  private addThumbnail (row: SQLRow, videoModel: VideoModel) {
+    const id = row['Thumbnails.id']
+    if (!id || this.thumbnailsDone.has(id)) return
 
-    const attributes = pick(row.Thumbnails, this.videoAttributes.getThumbnailAttributes())
+    const attributes = this.grab(row, this.tables.getThumbnailAttributes(), 'Thumbnails')
     const thumbnailModel = new ThumbnailModel(attributes, this.buildOpts)
     videoModel.Thumbnails.push(thumbnailModel)
 
-    this.thumbnailsDone.add(row.Thumbnails.id)
+    this.thumbnailsDone.add(id)
   }
 
-  private addWebTorrentFile (row: any, videoModel: VideoModel) {
-    if (!row.VideoFiles?.id || this.videoFileMemo[row.VideoFiles.id]) return
+  private addWebTorrentFile (row: SQLRow, videoModel: VideoModel) {
+    const id = row['VideoFiles.id']
+    if (!id || this.videoFileMemo[id]) return
 
-    const attributes = pick(row.VideoFiles, this.videoAttributes.getFileAttributes())
+    const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoFiles')
     const videoFileModel = new VideoFileModel(attributes, this.buildOpts)
     videoModel.VideoFiles.push(videoFileModel)
 
-    this.videoFileMemo[row.VideoFiles.id] = videoFileModel
+    this.videoFileMemo[id] = videoFileModel
   }
 
-  private addStreamingPlaylist (row: any, videoModel: VideoModel) {
-    if (!row.VideoStreamingPlaylists?.id || this.videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id]) return
+  private addStreamingPlaylist (row: SQLRow, videoModel: VideoModel) {
+    const id = row['VideoStreamingPlaylists.id']
+    if (!id || this.videoStreamingPlaylistMemo[id]) return
 
-    const attributes = pick(row.VideoStreamingPlaylists, this.videoAttributes.getStreamingPlaylistAttributes())
+    const attributes = this.grab(row, this.tables.getStreamingPlaylistAttributes(), 'VideoStreamingPlaylists')
     const streamingPlaylist = new VideoStreamingPlaylistModel(attributes, this.buildOpts)
     streamingPlaylist.VideoFiles = []
 
     videoModel.VideoStreamingPlaylists.push(streamingPlaylist)
 
-    this.videoStreamingPlaylistMemo[streamingPlaylist.id] = streamingPlaylist
+    this.videoStreamingPlaylistMemo[id] = streamingPlaylist
   }
 
-  private addStreamingPlaylistFile (row: any) {
-    if (!row.VideoStreamingPlaylists?.VideoFiles?.id || this.videoFileMemo[row.VideoStreamingPlaylists.VideoFiles.id]) return
+  private addStreamingPlaylistFile (row: SQLRow) {
+    const id = row['VideoStreamingPlaylists.VideoFiles.id']
+    if (!id || this.videoFileMemo[id]) return
 
-    const streamingPlaylist = this.videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id]
+    const streamingPlaylist = this.videoStreamingPlaylistMemo[row['VideoStreamingPlaylists.id']]
 
-    const attributes = pick(row.VideoStreamingPlaylists.VideoFiles, this.videoAttributes.getFileAttributes())
+    const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoStreamingPlaylists.VideoFiles')
     const videoFileModel = new VideoFileModel(attributes, this.buildOpts)
     streamingPlaylist.VideoFiles.push(videoFileModel)
 
-    this.videoFileMemo[row.VideoStreamingPlaylists.VideoFiles.id] = videoFileModel
+    this.videoFileMemo[id] = videoFileModel
   }
 
-  private addRedundancy (redundancyRow: any, to: VideoFileModel | VideoStreamingPlaylistModel) {
+  private addRedundancy (row: SQLRow, prefix: string, to: VideoFileModel | VideoStreamingPlaylistModel) {
     if (!to.RedundancyVideos) to.RedundancyVideos = []
 
-    if (!redundancyRow?.id || this.redundancyDone.has(redundancyRow.id)) return
+    const redundancyPrefix = `${prefix}.RedundancyVideos`
+    const id = row[`${redundancyPrefix}.id`]
+
+    if (!id || this.redundancyDone.has(id)) return
 
-    const attributes = pick(redundancyRow, this.videoAttributes.getRedundancyAttributes())
+    const attributes = this.grab(row, this.tables.getRedundancyAttributes(), redundancyPrefix)
     const redundancyModel = new VideoRedundancyModel(attributes, this.buildOpts)
     to.RedundancyVideos.push(redundancyModel)
 
-    this.redundancyDone.add(redundancyRow.id)
+    this.redundancyDone.add(id)
   }
 
-  private addTag (row: any, videoModel: VideoModel) {
-    if (!row.Tags?.name) return
-    const association = row.Tags.VideoTagModel
+  private addTag (row: SQLRow, videoModel: VideoModel) {
+    if (!row['Tags.name']) return
 
-    const key = `${association.videoId}-${association.tagId}`
+    const key = `${row['Tags.VideoTagModel.videoId']}-${row['Tags.VideoTagModel.tagId']}`
     if (this.tagsDone.has(key)) return
 
-    const attributes = pick(row.Tags, this.videoAttributes.getTagAttributes())
+    const attributes = this.grab(row, this.tables.getTagAttributes(), 'Tags')
     const tagModel = new TagModel(attributes, this.buildOpts)
     videoModel.Tags.push(tagModel)
 
     this.tagsDone.add(key)
   }
 
-  private addTracker (row: any, videoModel: VideoModel) {
-    if (!row.Trackers?.id) return
-    const association = row.Trackers.VideoTrackerModel
+  private addTracker (row: SQLRow, videoModel: VideoModel) {
+    if (!row['Trackers.id']) return
 
-    const key = `${association.videoId}-${association.trackerId}`
+    const key = `${row['Trackers.VideoTrackerModel.videoId']}-${row['Trackers.VideoTrackerModel.trackerId']}`
     if (this.trackersDone.has(key)) return
 
-    const attributes = pick(row.Trackers, this.videoAttributes.getTrackerAttributes())
+    const attributes = this.grab(row, this.tables.getTrackerAttributes(), 'Trackers')
     const trackerModel = new TrackerModel(attributes, this.buildOpts)
     videoModel.Trackers.push(trackerModel)
 
     this.trackersDone.add(key)
   }
 
-  private setBlacklisted (row: any, videoModel: VideoModel) {
-    if (!row.VideoBlacklist?.id) return
-    if (this.blacklistDone.has(row.VideoBlacklist.id)) return
+  private setBlacklisted (row: SQLRow, videoModel: VideoModel) {
+    const id = row['VideoBlacklist.id']
+    if (!id || this.blacklistDone.has(id)) return
 
-    const attributes = pick(row.VideoBlacklist, this.videoAttributes.getBlacklistedAttributes())
+    const attributes = this.grab(row, this.tables.getBlacklistedAttributes(), 'VideoBlacklist')
     videoModel.VideoBlacklist = new VideoBlacklistModel(attributes, this.buildOpts)
 
-    this.blacklistDone.add(row.VideoBlacklist.id)
+    this.blacklistDone.add(id)
   }
 
-  private setScheduleVideoUpdate (row: any, videoModel: VideoModel) {
-    if (!row.ScheduleVideoUpdate?.id) return
-    if (this.scheduleVideoUpdateDone.has(row.ScheduleVideoUpdate.id)) return
+  private setScheduleVideoUpdate (row: SQLRow, videoModel: VideoModel) {
+    const id = row['ScheduleVideoUpdate.id']
+    if (!id || this.scheduleVideoUpdateDone.has(id)) return
 
-    const attributes = pick(row.ScheduleVideoUpdate, this.videoAttributes.getScheduleUpdateAttributes())
+    const attributes = this.grab(row, this.tables.getScheduleUpdateAttributes(), 'ScheduleVideoUpdate')
     videoModel.ScheduleVideoUpdate = new ScheduleVideoUpdateModel(attributes, this.buildOpts)
 
-    this.scheduleVideoUpdateDone.add(row.ScheduleVideoUpdate.id)
+    this.scheduleVideoUpdateDone.add(id)
   }
 
-  private setLive (row: any, videoModel: VideoModel) {
-    if (!row.VideoLive?.id) return
-    if (this.liveDone.has(row.VideoLive.id)) return
+  private setLive (row: SQLRow, videoModel: VideoModel) {
+    const id = row['VideoLive.id']
+    if (!id || this.liveDone.has(id)) return
 
-    const attributes = pick(row.VideoLive, this.videoAttributes.getLiveAttributes())
+    const attributes = this.grab(row, this.tables.getLiveAttributes(), 'VideoLive')
     videoModel.VideoLive = new VideoLiveModel(attributes, this.buildOpts)
 
-    this.liveDone.add(row.ScheduleVideoUpdate.id)
+    this.liveDone.add(id)
+  }
+
+  private grab (row: SQLRow, attributes: string[], prefix: string) {
+    const result: { [ id: string ]: string | number } = {}
+
+    for (const a of attributes) {
+      const key = prefix
+        ? prefix + '.' + a
+        : a
+
+      result[a] = row[key]
+    }
+
+    return result
   }
 }