From c795e19663a93c24908a7318975f820bac63164f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 3 Aug 2022 15:08:36 +0200 Subject: [PATCH 1/1] Automatically rebuild native modules on ABI change --- server.ts | 6 ++ server/helpers/utils.ts | 26 +------- server/helpers/version.ts | 36 ++++++++++ server/initializers/constants.ts | 2 +- server/initializers/installer.ts | 7 +- .../migrations/0725-node-version.ts | 66 +++++++++++++++++++ server/lib/plugins/plugin-manager.ts | 9 ++- server/lib/plugins/yarn.ts | 5 ++ server/lib/server-config-manager.ts | 2 +- server/models/application/application.ts | 22 +++++++ server/tests/api/server/plugins.ts | 32 +++++++++ .../peertube-plugin-test-native/main.js | 21 ++++++ .../peertube-plugin-test-native/package.json | 23 +++++++ 13 files changed, 227 insertions(+), 30 deletions(-) create mode 100644 server/helpers/version.ts create mode 100644 server/initializers/migrations/0725-node-version.ts create mode 100644 server/tests/fixtures/peertube-plugin-test-native/main.js create mode 100644 server/tests/fixtures/peertube-plugin-test-native/package.json diff --git a/server.ts b/server.ts index 35ccc6758..aaf1ea021 100644 --- a/server.ts +++ b/server.ts @@ -138,6 +138,7 @@ import { ServerConfigManager } from '@server/lib/server-config-manager' import { VideoViewsManager } from '@server/lib/views/video-views-manager' import { isTestOrDevInstance } from './server/helpers/core-utils' import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics' +import { ApplicationModel } from '@server/models/application/application' // ----------- Command line ----------- @@ -330,12 +331,17 @@ async function startApplication () { server.listen(port, hostname, async () => { if (cliOptions.plugins) { try { + await PluginManager.Instance.rebuildNativePluginsIfNeeded() + await PluginManager.Instance.registerPluginsAndThemes() } catch (err) { logger.error('Cannot register plugins and themes.', { err }) } } + ApplicationModel.updateNodeVersions() + .catch(err => logger.error('Cannot update node versions.', { err })) + logger.info('HTTP server listening on %s:%d', hostname, port) logger.info('Web server: %s', WEBSERVER.URL) diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 6b9333b53..5a4fe4fdd 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -4,7 +4,7 @@ import { join } from 'path' import { sha256 } from '@shared/extra-utils' import { ResultList } from '@shared/models' import { CONFIG } from '../initializers/config' -import { execPromise, execPromise2, randomBytesPromise } from './core-utils' +import { randomBytesPromise } from './core-utils' import { logger } from './logger' function deleteFileAndCatch (path: string) { @@ -44,29 +44,6 @@ function getSecureTorrentName (originalName: string) { return sha256(originalName) + '.torrent' } -async function getServerCommit () { - try { - const tag = await execPromise2( - '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true', - { stdio: [ 0, 1, 2 ] } - ) - - if (tag) return tag.replace(/^v/, '') - } catch (err) { - logger.debug('Cannot get version from git tags.', { err }) - } - - try { - const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD') - - if (version) return version.toString().trim() - } catch (err) { - logger.debug('Cannot get version from git HEAD.', { err }) - } - - return '' -} - /** * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does @@ -88,7 +65,6 @@ export { generateRandomString, getFormattedObjects, getSecureTorrentName, - getServerCommit, generateVideoImportTmpPath, getUUIDFromFilename } diff --git a/server/helpers/version.ts b/server/helpers/version.ts new file mode 100644 index 000000000..5b3bf59dd --- /dev/null +++ b/server/helpers/version.ts @@ -0,0 +1,36 @@ +import { execPromise, execPromise2 } from './core-utils' +import { logger } from './logger' + +async function getServerCommit () { + try { + const tag = await execPromise2( + '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true', + { stdio: [ 0, 1, 2 ] } + ) + + if (tag) return tag.replace(/^v/, '') + } catch (err) { + logger.debug('Cannot get version from git tags.', { err }) + } + + try { + const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD') + + if (version) return version.toString().trim() + } catch (err) { + logger.debug('Cannot get version from git HEAD.', { err }) + } + + return '' +} + +function getNodeABIVersion () { + const version = process.versions.modules + + return parseInt(version) +} + +export { + getServerCommit, + getNodeABIVersion +} diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 99ae64f8d..8165a289d 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 720 +const LAST_MIGRATION_VERSION = 725 // --------------------------------------------------------------------------- diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index 7d5919459..b02be9567 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -1,6 +1,8 @@ import { ensureDir, readdir, remove } from 'fs-extra' import passwordGenerator from 'password-generator' import { join } from 'path' +import { isTestOrDevInstance } from '@server/helpers/core-utils' +import { getNodeABIVersion } from '@server/helpers/version' import { UserRole } from '@shared/models' import { logger } from '../helpers/logger' import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user' @@ -10,7 +12,6 @@ import { applicationExist, clientsExist, usersExist } from './checker-after-init import { CONFIG } from './config' import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants' import { sequelizeTypescript } from './database' -import { isTestOrDevInstance } from '@server/helpers/core-utils' async function installApplication () { try { @@ -175,7 +176,9 @@ async function createApplicationIfNotExist () { logger.info('Creating application account.') const application = await ApplicationModel.create({ - migrationVersion: LAST_MIGRATION_VERSION + migrationVersion: LAST_MIGRATION_VERSION, + nodeVersion: process.version, + nodeABIVersion: getNodeABIVersion() }) return createApplicationActor(application.id) diff --git a/server/initializers/migrations/0725-node-version.ts b/server/initializers/migrations/0725-node-version.ts new file mode 100644 index 000000000..d8b9cc750 --- /dev/null +++ b/server/initializers/migrations/0725-node-version.ts @@ -0,0 +1,66 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + const { transaction } = utils + + { + const data = { + type: Sequelize.STRING, + defaultValue: null, + allowNull: true + } + await utils.queryInterface.addColumn('application', 'nodeVersion', data, { transaction }) + } + + { + const data = { + type: Sequelize.STRING, + defaultValue: null, + allowNull: true + } + await utils.queryInterface.addColumn('application', 'nodeABIVersion', data, { transaction }) + } + + { + const query = `UPDATE "application" SET "nodeVersion" = '${process.version}'` + await utils.sequelize.query(query, { transaction }) + } + + { + const nodeABIVersion = parseInt(process.versions.modules) + const query = `UPDATE "application" SET "nodeABIVersion" = ${nodeABIVersion}` + await utils.sequelize.query(query, { transaction }) + } + + { + const data = { + type: Sequelize.STRING, + defaultValue: null, + allowNull: false + } + await utils.queryInterface.changeColumn('application', 'nodeVersion', data, { transaction }) + } + + { + const data = { + type: Sequelize.STRING, + defaultValue: null, + allowNull: false + } + await utils.queryInterface.changeColumn('application', 'nodeABIVersion', data, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts index a706df1e0..a46b97fa4 100644 --- a/server/lib/plugins/plugin-manager.ts +++ b/server/lib/plugins/plugin-manager.ts @@ -3,6 +3,7 @@ import { createReadStream, createWriteStream } from 'fs' import { ensureDir, outputFile, readJSON } from 'fs-extra' import { basename, join } from 'path' import { decachePlugin } from '@server/helpers/decache' +import { ApplicationModel } from '@server/models/application/application' import { MOAuthTokenUser, MUser } from '@server/types/models' import { getCompleteLocale } from '@shared/core-utils' import { @@ -23,7 +24,7 @@ import { PluginModel } from '../../models/server/plugin' import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPassOptions, RegisterServerOptions } from '../../types/plugins' import { ClientHtml } from '../client-html' import { RegisterHelpers } from './register-helpers' -import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn' +import { installNpmPlugin, installNpmPluginFromDisk, rebuildNativePlugins, removeNpmPlugin } from './yarn' export interface RegisteredPlugin { npmName: string @@ -384,6 +385,12 @@ export class PluginManager implements ServerHook { logger.info('Plugin %s uninstalled.', npmName) } + async rebuildNativePluginsIfNeeded () { + if (!await ApplicationModel.nodeABIChanged()) return + + return rebuildNativePlugins() + } + // ###################### Private register ###################### private async registerPluginOrTheme (plugin: PluginModel) { diff --git a/server/lib/plugins/yarn.ts b/server/lib/plugins/yarn.ts index 3f45681d3..d105b95e0 100644 --- a/server/lib/plugins/yarn.ts +++ b/server/lib/plugins/yarn.ts @@ -31,11 +31,16 @@ async function removeNpmPlugin (name: string) { await execYarn('remove ' + name) } +async function rebuildNativePlugins () { + await execYarn('install --pure-lockfile') +} + // ############################################################################ export { installNpmPlugin, installNpmPluginFromDisk, + rebuildNativePlugins, removeNpmPlugin } diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index d16a88f65..a3312fa20 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts @@ -1,4 +1,4 @@ -import { getServerCommit } from '@server/helpers/utils' +import { getServerCommit } from '@server/helpers/version' import { CONFIG, isEmailEnabled } from '@server/initializers/config' import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants' import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup' diff --git a/server/models/application/application.ts b/server/models/application/application.ts index a479de5d2..d4590e001 100644 --- a/server/models/application/application.ts +++ b/server/models/application/application.ts @@ -1,5 +1,6 @@ import memoizee from 'memoizee' import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Model, Table } from 'sequelize-typescript' +import { getNodeABIVersion } from '@server/helpers/version' import { AttributesOnly } from '@shared/typescript-utils' import { AccountModel } from '../account/account' @@ -37,6 +38,14 @@ export class ApplicationModel extends Model AccountModel, { foreignKey: { allowNull: true @@ -52,4 +61,17 @@ export class ApplicationModel extends Model { + print('hello world') + res.sendStatus(204) + }) +} + +async function unregister () { + return +} + +module.exports = { + register, + unregister +} diff --git a/server/tests/fixtures/peertube-plugin-test-native/package.json b/server/tests/fixtures/peertube-plugin-test-native/package.json new file mode 100644 index 000000000..a6525720b --- /dev/null +++ b/server/tests/fixtures/peertube-plugin-test-native/package.json @@ -0,0 +1,23 @@ +{ + "name": "peertube-plugin-test-native", + "version": "0.0.1", + "description": "Plugin test-native", + "engine": { + "peertube": ">=4.3.0" + }, + "keywords": [ + "peertube", + "plugin" + ], + "homepage": "https://github.com/Chocobozzz/PeerTube", + "author": "Chocobozzz", + "bugs": "https://github.com/Chocobozzz/PeerTube/issues", + "library": "./main.js", + "staticDirs": {}, + "css": [], + "clientScripts": [], + "translations": {}, + "dependencies": { + "a-native-example": "^1.0.0" + } +} -- 2.41.0