From 6702a1b2ccd666285dee9c72b5bace641d2fce8b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 16 Jul 2019 11:33:22 +0200 Subject: Add ability to search available plugins --- server/controllers/api/plugins.ts | 28 +++++++++-- server/initializers/config.ts | 6 +++ server/initializers/constants.ts | 5 +- server/lib/plugins/plugin-index.ts | 64 ++++++++++++++++++++++++ server/lib/plugins/plugin-manager.ts | 4 ++ server/lib/schedulers/plugins-check-scheduler.ts | 60 ++++++++++++++++++++++ server/middlewares/validators/plugins.ts | 30 ++++++++++- server/middlewares/validators/sort.ts | 3 ++ server/models/server/plugin.ts | 19 +++++-- 9 files changed, 210 insertions(+), 9 deletions(-) create mode 100644 server/lib/plugins/plugin-index.ts create mode 100644 server/lib/schedulers/plugins-check-scheduler.ts (limited to 'server') diff --git a/server/controllers/api/plugins.ts b/server/controllers/api/plugins.ts index 14675fdf3..114cc49b6 100644 --- a/server/controllers/api/plugins.ts +++ b/server/controllers/api/plugins.ts @@ -8,12 +8,13 @@ import { setDefaultPagination, setDefaultSort } from '../../middlewares' -import { pluginsSortValidator } from '../../middlewares/validators' +import { availablePluginsSortValidator, pluginsSortValidator } from '../../middlewares/validators' import { PluginModel } from '../../models/server/plugin' import { UserRight } from '../../../shared/models/users' import { existingPluginValidator, installOrUpdatePluginValidator, + listAvailablePluginsValidator, listPluginsValidator, uninstallPluginValidator, updatePluginSettingsValidator @@ -22,9 +23,22 @@ import { PluginManager } from '../../lib/plugins/plugin-manager' import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/install-plugin.model' import { ManagePlugin } from '../../../shared/models/plugins/manage-plugin.model' import { logger } from '../../helpers/logger' +import { listAvailablePluginsFromIndex } from '../../lib/plugins/plugin-index' +import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' const pluginRouter = express.Router() +pluginRouter.get('/available', + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + listAvailablePluginsValidator, + paginationValidator, + availablePluginsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listAvailablePlugins) +) + pluginRouter.get('/', authenticate, ensureUserHasRight(UserRight.MANAGE_PLUGINS), @@ -88,10 +102,10 @@ export { // --------------------------------------------------------------------------- async function listPlugins (req: express.Request, res: express.Response) { - const type = req.query.type + const pluginType = req.query.pluginType const resultList = await PluginModel.listForApi({ - type, + pluginType, start: req.query.start, count: req.query.count, sort: req.query.sort @@ -160,3 +174,11 @@ async function updatePluginSettings (req: express.Request, res: express.Response return res.sendStatus(204) } + +async function listAvailablePlugins (req: express.Request, res: express.Response) { + const query: PeertubePluginIndexList = req.query + + const resultList = await listAvailablePluginsFromIndex(query) + + return res.json(resultList) +} diff --git a/server/initializers/config.ts b/server/initializers/config.ts index dfc4bea21..2c1b30021 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -134,6 +134,12 @@ const CONFIG = { } } }, + PLUGINS: { + INDEX: { + ENABLED: config.get('plugins.index.enabled'), + URL: config.get('plugins.index.url') + } + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 2d487a263..06e8c070b 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -64,7 +64,9 @@ const SORTABLE_COLUMNS = { VIDEO_PLAYLISTS: [ 'displayName', 'createdAt', 'updatedAt' ], - PLUGINS: [ 'name', 'createdAt', 'updatedAt' ] + PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], + + AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ] } const OAUTH_LIFETIME = { @@ -165,6 +167,7 @@ const SCHEDULER_INTERVALS_MS = { removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute youtubeDLUpdate: 60000 * 60 * 24, // 1 day + checkPlugins: 60000 * 60 * 24, // 1 day removeOldViews: 60000 * 60 * 24, // 1 day removeOldHistory: 60000 * 60 * 24 // 1 day } diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts new file mode 100644 index 000000000..4a8a90ec8 --- /dev/null +++ b/server/lib/plugins/plugin-index.ts @@ -0,0 +1,64 @@ +import { doRequest } from '../../helpers/requests' +import { CONFIG } from '../../initializers/config' +import { + PeertubePluginLatestVersionRequest, + PeertubePluginLatestVersionResponse +} from '../../../shared/models/plugins/peertube-plugin-latest-version.model' +import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' +import { ResultList } from '../../../shared/models' +import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' +import { PluginModel } from '../../models/server/plugin' +import { PluginManager } from './plugin-manager' +import { logger } from '../../helpers/logger' + +const packageJSON = require('../../../../package.json') + +async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { + const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options + + const qs: PeertubePluginIndexList = { + start, + count, + sort, + pluginType, + search, + currentPeerTubeEngine: packageJSON.version + } + + const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' + + const { body } = await doRequest({ uri, qs, json: true }) + + logger.debug('Got result from PeerTube index.', { body }) + + await addInstanceInformation(body) + + return body as ResultList +} + +async function addInstanceInformation (result: ResultList) { + for (const d of result.data) { + d.installed = PluginManager.Instance.isRegistered(d.npmName) + d.name = PluginModel.normalizePluginName(d.npmName) + } + + return result +} + +async function getLatestPluginsVersion (npmNames: string[]): Promise { + const bodyRequest: PeertubePluginLatestVersionRequest = { + npmNames, + currentPeerTubeEngine: packageJSON.version + } + + const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins/latest-version' + + const { body } = await doRequest({ uri, body: bodyRequest }) + + return body +} + +export { + listAvailablePluginsFromIndex, + getLatestPluginsVersion +} diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts index 7576b284c..9e4ec5adf 100644 --- a/server/lib/plugins/plugin-manager.ts +++ b/server/lib/plugins/plugin-manager.ts @@ -55,6 +55,10 @@ export class PluginManager { // ###################### Getters ###################### + isRegistered (npmName: string) { + return !!this.getRegisteredPluginOrTheme(npmName) + } + getRegisteredPluginOrTheme (npmName: string) { return this.registeredPlugins[npmName] } diff --git a/server/lib/schedulers/plugins-check-scheduler.ts b/server/lib/schedulers/plugins-check-scheduler.ts new file mode 100644 index 000000000..9c60dbcd4 --- /dev/null +++ b/server/lib/schedulers/plugins-check-scheduler.ts @@ -0,0 +1,60 @@ +import { logger } from '../../helpers/logger' +import { AbstractScheduler } from './abstract-scheduler' +import { retryTransactionWrapper } from '../../helpers/database-utils' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' +import { CONFIG } from '../../initializers/config' +import { PluginModel } from '../../models/server/plugin' +import { chunk } from 'lodash' +import { getLatestPluginsVersion } from '../plugins/plugin-index' +import { compareSemVer } from '../../../shared/core-utils/miscs/miscs' + +export class PluginsCheckScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.checkPlugins + + private constructor () { + super() + } + + protected async internalExecute () { + return retryTransactionWrapper(this.checkLatestPluginsVersion.bind(this)) + } + + private async checkLatestPluginsVersion () { + if (CONFIG.PLUGINS.INDEX.ENABLED === false) return + + logger.info('Checkin latest plugins version.') + + const plugins = await PluginModel.listInstalled() + + // Process 10 plugins in 1 HTTP request + const chunks = chunk(plugins, 10) + for (const chunk of chunks) { + // Find plugins according to their npm name + const pluginIndex: { [npmName: string]: PluginModel} = {} + for (const plugin of chunk) { + pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin + } + + const npmNames = Object.keys(pluginIndex) + const results = await getLatestPluginsVersion(npmNames) + + for (const result of results) { + const plugin = pluginIndex[result.npmName] + if (!result.latestVersion) continue + + if (plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0) { + plugin.latestVersion = result.latestVersion + await plugin.save() + } + } + } + + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/middlewares/validators/plugins.ts b/server/middlewares/validators/plugins.ts index 8103ec7d3..8cb3153aa 100644 --- a/server/middlewares/validators/plugins.ts +++ b/server/middlewares/validators/plugins.ts @@ -8,6 +8,7 @@ import { isBooleanValid, isSafePath } from '../../helpers/custom-validators/misc import { PluginModel } from '../../models/server/plugin' import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/install-plugin.model' import { PluginType } from '../../../shared/models/plugins/plugin.type' +import { CONFIG } from '../../initializers/config' const servePluginStaticDirectoryValidator = (pluginType: PluginType) => [ param('pluginName').custom(isPluginNameValid).withMessage('Should have a valid plugin name'), @@ -33,7 +34,7 @@ const servePluginStaticDirectoryValidator = (pluginType: PluginType) => [ ] const listPluginsValidator = [ - query('type') + query('pluginType') .optional() .custom(isPluginTypeValid).withMessage('Should have a valid plugin type'), query('uninstalled') @@ -119,12 +120,39 @@ const updatePluginSettingsValidator = [ } ] +const listAvailablePluginsValidator = [ + query('sort') + .optional() + .exists().withMessage('Should have a valid sort'), + query('search') + .optional() + .exists().withMessage('Should have a valid search'), + query('pluginType') + .optional() + .custom(isPluginTypeValid).withMessage('Should have a valid plugin type'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking enabledPluginValidator parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + if (CONFIG.PLUGINS.INDEX.ENABLED === false) { + return res.status(400) + .json({ error: 'Plugin index is not enabled' }) + .end() + } + + return next() + } +] + // --------------------------------------------------------------------------- export { servePluginStaticDirectoryValidator, updatePluginSettingsValidator, uninstallPluginValidator, + listAvailablePluginsValidator, existingPluginValidator, installOrUpdatePluginValidator, listPluginsValidator diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index 102db85cb..c75e701d6 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts @@ -22,6 +22,7 @@ const SORTABLE_SERVERS_BLOCKLIST_COLUMNS = createSortableColumns(SORTABLE_COLUMN const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.USER_NOTIFICATIONS) const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS) +const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS) const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) @@ -43,6 +44,7 @@ const serversBlocklistSortValidator = checkSort(SORTABLE_SERVERS_BLOCKLIST_COLUM const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COLUMNS) const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS) const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS) +const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS) // --------------------------------------------------------------------------- @@ -61,6 +63,7 @@ export { videoCommentThreadsSortValidator, videoRatesSortValidator, userSubscriptionsSortValidator, + availablePluginsSortValidator, videoChannelsSearchSortValidator, accountsBlocklistSortValidator, serversBlocklistSortValidator, diff --git a/server/models/server/plugin.ts b/server/models/server/plugin.ts index bd3d7a81e..ba43713f6 100644 --- a/server/models/server/plugin.ts +++ b/server/models/server/plugin.ts @@ -10,6 +10,7 @@ import { import { PluginType } from '../../../shared/models/plugins/plugin.type' import { PeerTubePlugin } from '../../../shared/models/plugins/peertube-plugin.model' import { FindAndCountOptions, json } from 'sequelize' +import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' @DefaultScope(() => ({ attributes: { @@ -177,7 +178,7 @@ export class PluginModel extends Model { } static listForApi (options: { - type?: PluginType, + pluginType?: PluginType, uninstalled?: boolean, start: number, count: number, @@ -193,7 +194,7 @@ export class PluginModel extends Model { } } - if (options.type) query.where['type'] = options.type + if (options.pluginType) query.where['type'] = options.pluginType return PluginModel .findAndCountAll(query) @@ -202,8 +203,18 @@ export class PluginModel extends Model { }) } - static normalizePluginName (name: string) { - return name.replace(/^peertube-((theme)|(plugin))-/, '') + static listInstalled () { + const query = { + where: { + uninstalled: false + } + } + + return PluginModel.findAll(query) + } + + static normalizePluginName (npmName: string) { + return npmName.replace(/^peertube-((theme)|(plugin))-/, '') } static getTypeFromNpmName (npmName: string) { -- cgit v1.2.3