Add ability to search available plugins

9 files changed, 210 insertions, 9 deletions

diff --git a/server/controllers/api/plugins.ts b/server/controllers/api/plugins.ts
index 14675fdf3..114cc49b6 100644
--- a/server/controllers/api/plugins.ts
+++ b/server/controllers/api/plugins.ts
@@ -8,12 +8,13 @@ import {
   setDefaultPagination,
   setDefaultSort
 } from '../../middlewares'
-import { pluginsSortValidator } from '../../middlewares/validators'
+import { availablePluginsSortValidator, pluginsSortValidator } from '../../middlewares/validators'
 import { PluginModel } from '../../models/server/plugin'
 import { UserRight } from '../../../shared/models/users'
 import {
   existingPluginValidator,
   installOrUpdatePluginValidator,
+  listAvailablePluginsValidator,
   listPluginsValidator,
   uninstallPluginValidator,
   updatePluginSettingsValidator
@@ -22,9 +23,22 @@ import { PluginManager } from '../../lib/plugins/plugin-manager'
 import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/install-plugin.model'
 import { ManagePlugin } from '../../../shared/models/plugins/manage-plugin.model'
 import { logger } from '../../helpers/logger'
+import { listAvailablePluginsFromIndex } from '../../lib/plugins/plugin-index'
+import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model'
 
 const pluginRouter = express.Router()
 
+pluginRouter.get('/available',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_PLUGINS),
+  listAvailablePluginsValidator,
+  paginationValidator,
+  availablePluginsSortValidator,
+  setDefaultSort,
+  setDefaultPagination,
+  asyncMiddleware(listAvailablePlugins)
+)
+
 pluginRouter.get('/',
   authenticate,
   ensureUserHasRight(UserRight.MANAGE_PLUGINS),
@@ -88,10 +102,10 @@ export {
 // ---------------------------------------------------------------------------
 
 async function listPlugins (req: express.Request, res: express.Response) {
-  const type = req.query.type
+  const pluginType = req.query.pluginType
 
   const resultList = await PluginModel.listForApi({
-    type,
+    pluginType,
     start: req.query.start,
     count: req.query.count,
     sort: req.query.sort
@@ -160,3 +174,11 @@ async function updatePluginSettings (req: express.Request, res: express.Response
 
   return res.sendStatus(204)
 }
+
+async function listAvailablePlugins (req: express.Request, res: express.Response) {
+  const query: PeertubePluginIndexList = req.query
+
+  const resultList = await listAvailablePluginsFromIndex(query)
+
+  return res.json(resultList)
+}
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index dfc4bea21..2c1b30021 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -134,6 +134,12 @@ const CONFIG = {
       }
     }
   },
+  PLUGINS: {
+    INDEX: {
+      ENABLED: config.get<boolean>('plugins.index.enabled'),
+      URL: config.get<boolean>('plugins.index.url')
+    }
+  },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 2d487a263..06e8c070b 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -64,7 +64,9 @@ const SORTABLE_COLUMNS = {
 
   VIDEO_PLAYLISTS: [ 'displayName', 'createdAt', 'updatedAt' ],
 
-  PLUGINS: [ 'name', 'createdAt', 'updatedAt' ]
+  PLUGINS: [ 'name', 'createdAt', 'updatedAt' ],
+
+  AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ]
 }
 
 const OAUTH_LIFETIME = {
@@ -165,6 +167,7 @@ const SCHEDULER_INTERVALS_MS = {
   removeOldJobs: 60000 * 60, // 1 hour
   updateVideos: 60000, // 1 minute
   youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+  checkPlugins: 60000 * 60 * 24, // 1 day
   removeOldViews: 60000 * 60 * 24, // 1 day
   removeOldHistory: 60000 * 60 * 24 // 1 day
 }
diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts
new file mode 100644
index 000000000..4a8a90ec8
--- /dev/null
+++ b/server/lib/plugins/plugin-index.ts
@@ -0,0 +1,64 @@
+import { doRequest } from '../../helpers/requests'
+import { CONFIG } from '../../initializers/config'
+import {
+  PeertubePluginLatestVersionRequest,
+  PeertubePluginLatestVersionResponse
+} from '../../../shared/models/plugins/peertube-plugin-latest-version.model'
+import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model'
+import { ResultList } from '../../../shared/models'
+import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model'
+import { PluginModel } from '../../models/server/plugin'
+import { PluginManager } from './plugin-manager'
+import { logger } from '../../helpers/logger'
+
+const packageJSON = require('../../../../package.json')
+
+async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) {
+  const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options
+
+  const qs: PeertubePluginIndexList = {
+    start,
+    count,
+    sort,
+    pluginType,
+    search,
+    currentPeerTubeEngine: packageJSON.version
+  }
+
+  const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins'
+
+  const { body } = await doRequest({ uri, qs, json: true })
+
+  logger.debug('Got result from PeerTube index.', { body })
+
+  await addInstanceInformation(body)
+
+  return body as ResultList<PeerTubePluginIndex>
+}
+
+async function addInstanceInformation (result: ResultList<PeerTubePluginIndex>) {
+  for (const d of result.data) {
+    d.installed = PluginManager.Instance.isRegistered(d.npmName)
+    d.name = PluginModel.normalizePluginName(d.npmName)
+  }
+
+  return result
+}
+
+async function getLatestPluginsVersion (npmNames: string[]): Promise<PeertubePluginLatestVersionResponse> {
+  const bodyRequest: PeertubePluginLatestVersionRequest = {
+    npmNames,
+    currentPeerTubeEngine: packageJSON.version
+  }
+
+  const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins/latest-version'
+
+  const { body } = await doRequest({ uri, body: bodyRequest })
+
+  return body
+}
+
+export {
+  listAvailablePluginsFromIndex,
+  getLatestPluginsVersion
+}
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts
index 7576b284c..9e4ec5adf 100644
--- a/server/lib/plugins/plugin-manager.ts
+++ b/server/lib/plugins/plugin-manager.ts
@@ -55,6 +55,10 @@ export class PluginManager {
 
   // ###################### Getters ######################
 
+  isRegistered (npmName: string) {
+    return !!this.getRegisteredPluginOrTheme(npmName)
+  }
+
   getRegisteredPluginOrTheme (npmName: string) {
     return this.registeredPlugins[npmName]
   }
diff --git a/server/lib/schedulers/plugins-check-scheduler.ts b/server/lib/schedulers/plugins-check-scheduler.ts
new file mode 100644
index 000000000..9c60dbcd4
--- /dev/null
+++ b/server/lib/schedulers/plugins-check-scheduler.ts
@@ -0,0 +1,60 @@
+import { logger } from '../../helpers/logger'
+import { AbstractScheduler } from './abstract-scheduler'
+import { retryTransactionWrapper } from '../../helpers/database-utils'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { CONFIG } from '../../initializers/config'
+import { PluginModel } from '../../models/server/plugin'
+import { chunk } from 'lodash'
+import { getLatestPluginsVersion } from '../plugins/plugin-index'
+import { compareSemVer } from '../../../shared/core-utils/miscs/miscs'
+
+export class PluginsCheckScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.checkPlugins
+
+  private constructor () {
+    super()
+  }
+
+  protected async internalExecute () {
+    return retryTransactionWrapper(this.checkLatestPluginsVersion.bind(this))
+  }
+
+  private async checkLatestPluginsVersion () {
+    if (CONFIG.PLUGINS.INDEX.ENABLED === false) return
+
+    logger.info('Checkin latest plugins version.')
+
+    const plugins = await PluginModel.listInstalled()
+
+    // Process 10 plugins in 1 HTTP request
+    const chunks = chunk(plugins, 10)
+    for (const chunk of chunks) {
+      // Find plugins according to their npm name
+      const pluginIndex: { [npmName: string]: PluginModel} = {}
+      for (const plugin of chunk) {
+        pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin
+      }
+
+      const npmNames = Object.keys(pluginIndex)
+      const results = await getLatestPluginsVersion(npmNames)
+
+      for (const result of results) {
+        const plugin = pluginIndex[result.npmName]
+        if (!result.latestVersion) continue
+
+        if (plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0) {
+          plugin.latestVersion = result.latestVersion
+          await plugin.save()
+        }
+      }
+    }
+
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
diff --git a/server/middlewares/validators/plugins.ts b/server/middlewares/validators/plugins.ts
index 8103ec7d3..8cb3153aa 100644
--- a/server/middlewares/validators/plugins.ts
+++ b/server/middlewares/validators/plugins.ts
@@ -8,6 +8,7 @@ import { isBooleanValid, isSafePath } from '../../helpers/custom-validators/misc
 import { PluginModel } from '../../models/server/plugin'
 import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/install-plugin.model'
 import { PluginType } from '../../../shared/models/plugins/plugin.type'
+import { CONFIG } from '../../initializers/config'
 
 const servePluginStaticDirectoryValidator = (pluginType: PluginType) => [
   param('pluginName').custom(isPluginNameValid).withMessage('Should have a valid plugin name'),
@@ -33,7 +34,7 @@ const servePluginStaticDirectoryValidator = (pluginType: PluginType) => [
 ]
 
 const listPluginsValidator = [
-  query('type')
+  query('pluginType')
     .optional()
     .custom(isPluginTypeValid).withMessage('Should have a valid plugin type'),
   query('uninstalled')
@@ -119,12 +120,39 @@ const updatePluginSettingsValidator = [
   }
 ]
 
+const listAvailablePluginsValidator = [
+  query('sort')
+    .optional()
+    .exists().withMessage('Should have a valid sort'),
+  query('search')
+    .optional()
+    .exists().withMessage('Should have a valid search'),
+  query('pluginType')
+    .optional()
+    .custom(isPluginTypeValid).withMessage('Should have a valid plugin type'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking enabledPluginValidator parameters', { parameters: req.query })
+
+    if (areValidationErrors(req, res)) return
+
+    if (CONFIG.PLUGINS.INDEX.ENABLED === false) {
+      return res.status(400)
+        .json({ error: 'Plugin index is not enabled' })
+        .end()
+    }
+
+    return next()
+  }
+]
+
 // ---------------------------------------------------------------------------
 
 export {
   servePluginStaticDirectoryValidator,
   updatePluginSettingsValidator,
   uninstallPluginValidator,
+  listAvailablePluginsValidator,
   existingPluginValidator,
   installOrUpdatePluginValidator,
   listPluginsValidator
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index 102db85cb..c75e701d6 100644
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -22,6 +22,7 @@ const SORTABLE_SERVERS_BLOCKLIST_COLUMNS = createSortableColumns(SORTABLE_COLUMN
 const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.USER_NOTIFICATIONS)
 const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS)
 const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS)
+const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
 
 const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS)
 const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS)
@@ -43,6 +44,7 @@ const serversBlocklistSortValidator = checkSort(SORTABLE_SERVERS_BLOCKLIST_COLUM
 const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COLUMNS)
 const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS)
 const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS)
+const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS)
 
 // ---------------------------------------------------------------------------
 
@@ -61,6 +63,7 @@ export {
   videoCommentThreadsSortValidator,
   videoRatesSortValidator,
   userSubscriptionsSortValidator,
+  availablePluginsSortValidator,
   videoChannelsSearchSortValidator,
   accountsBlocklistSortValidator,
   serversBlocklistSortValidator,
diff --git a/server/models/server/plugin.ts b/server/models/server/plugin.ts
index bd3d7a81e..ba43713f6 100644
--- a/server/models/server/plugin.ts
+++ b/server/models/server/plugin.ts
@@ -10,6 +10,7 @@ import {
 import { PluginType } from '../../../shared/models/plugins/plugin.type'
 import { PeerTubePlugin } from '../../../shared/models/plugins/peertube-plugin.model'
 import { FindAndCountOptions, json } from 'sequelize'
+import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model'
 
 @DefaultScope(() => ({
   attributes: {
@@ -177,7 +178,7 @@ export class PluginModel extends Model<PluginModel> {
   }
 
   static listForApi (options: {
-    type?: PluginType,
+    pluginType?: PluginType,
     uninstalled?: boolean,
     start: number,
     count: number,
@@ -193,7 +194,7 @@ export class PluginModel extends Model<PluginModel> {
       }
     }
 
-    if (options.type) query.where['type'] = options.type
+    if (options.pluginType) query.where['type'] = options.pluginType
 
     return PluginModel
       .findAndCountAll(query)
@@ -202,8 +203,18 @@ export class PluginModel extends Model<PluginModel> {
       })
   }
 
-  static normalizePluginName (name: string) {
-    return name.replace(/^peertube-((theme)|(plugin))-/, '')
+  static listInstalled () {
+    const query = {
+      where: {
+        uninstalled: false
+      }
+    }
+
+    return PluginModel.findAll(query)
+  }
+
+  static normalizePluginName (npmName: string) {
+    return npmName.replace(/^peertube-((theme)|(plugin))-/, '')
   }
 
   static getTypeFromNpmName (npmName: string) {