]>
Commit | Line | Data |
---|---|---|
8ebf2a5d | 1 | |
41fb13c3 C |
2 | import { mapSeries } from 'bluebird' |
3 | import { FSWatcher, watch } from 'chokidar' | |
8ebf2a5d C |
4 | import { FfmpegCommand } from 'fluent-ffmpeg' |
5 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | |
b3ce3606 | 6 | import PQueue from 'p-queue' |
8ebf2a5d C |
7 | import { basename, join } from 'path' |
8 | import { EventEmitter } from 'stream' | |
c729caf6 | 9 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg' |
8ebf2a5d C |
10 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' |
11 | import { CONFIG } from '@server/initializers/config' | |
12 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | |
aa887096 | 13 | import { removeHLSFileObjectStorageByPath, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage' |
8ebf2a5d C |
14 | import { VideoFileModel } from '@server/models/video/video-file' |
15 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | |
cfd57d2c | 16 | import { VideoStorage } from '@shared/models' |
4ec52d04 | 17 | import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' |
c729caf6 | 18 | import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' |
8ebf2a5d | 19 | import { isAbleToUploadVideo } from '../../user' |
8ebf2a5d C |
20 | import { LiveQuotaStore } from '../live-quota-store' |
21 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | |
22 | import { buildConcatenatedName } from '../live-utils' | |
23 | ||
24 | import memoizee = require('memoizee') | |
8ebf2a5d | 25 | interface MuxingSessionEvents { |
cfd57d2c | 26 | 'live-ready': (options: { videoId: number }) => void |
8ebf2a5d | 27 | |
15eb9e5b C |
28 | 'bad-socket-health': (options: { videoId: number }) => void |
29 | 'duration-exceeded': (options: { videoId: number }) => void | |
30 | 'quota-exceeded': (options: { videoId: number }) => void | |
8ebf2a5d | 31 | |
15eb9e5b C |
32 | 'ffmpeg-end': (options: { videoId: number }) => void |
33 | 'ffmpeg-error': (options: { videoId: number }) => void | |
8ebf2a5d | 34 | |
15eb9e5b | 35 | 'after-cleanup': (options: { videoId: number }) => void |
8ebf2a5d C |
36 | } |
37 | ||
38 | declare interface MuxingSession { | |
39 | on<U extends keyof MuxingSessionEvents>( | |
40 | event: U, listener: MuxingSessionEvents[U] | |
41 | ): this | |
42 | ||
43 | emit<U extends keyof MuxingSessionEvents>( | |
44 | event: U, ...args: Parameters<MuxingSessionEvents[U]> | |
45 | ): boolean | |
46 | } | |
47 | ||
48 | class MuxingSession extends EventEmitter { | |
49 | ||
50 | private ffmpegCommand: FfmpegCommand | |
51 | ||
52 | private readonly context: any | |
53 | private readonly user: MUserId | |
54 | private readonly sessionId: string | |
55 | private readonly videoLive: MVideoLiveVideo | |
56 | private readonly streamingPlaylist: MStreamingPlaylistVideo | |
df1db951 | 57 | private readonly inputUrl: string |
8ebf2a5d C |
58 | private readonly fps: number |
59 | private readonly allResolutions: number[] | |
60 | ||
679c12e6 C |
61 | private readonly bitrate: number |
62 | private readonly ratio: number | |
63 | ||
1ce4256a C |
64 | private readonly hasAudio: boolean |
65 | ||
8ebf2a5d C |
66 | private readonly videoId: number |
67 | private readonly videoUUID: string | |
68 | private readonly saveReplay: boolean | |
69 | ||
4ec52d04 C |
70 | private readonly outDirectory: string |
71 | private readonly replayDirectory: string | |
72 | ||
cfd57d2c C |
73 | private readonly liveSegmentShaStore: LiveSegmentShaStore |
74 | ||
8ebf2a5d C |
75 | private readonly lTags: LoggerTagsFn |
76 | ||
77 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | |
78 | ||
41fb13c3 C |
79 | private tsWatcher: FSWatcher |
80 | private masterWatcher: FSWatcher | |
cfd57d2c C |
81 | private m3u8Watcher: FSWatcher |
82 | ||
83 | private masterPlaylistCreated = false | |
84 | private liveReady = false | |
8ebf2a5d | 85 | |
39d117a4 C |
86 | private aborted = false |
87 | ||
8ebf2a5d C |
88 | private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { |
89 | return isAbleToUploadVideo(userId, 1000) | |
90 | }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) | |
91 | ||
92 | private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { | |
93 | return this.hasClientSocketInBadHealth(sessionId) | |
94 | }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) | |
95 | ||
96 | constructor (options: { | |
97 | context: any | |
98 | user: MUserId | |
99 | sessionId: string | |
100 | videoLive: MVideoLiveVideo | |
101 | streamingPlaylist: MStreamingPlaylistVideo | |
df1db951 | 102 | inputUrl: string |
8ebf2a5d | 103 | fps: number |
c826f34a | 104 | bitrate: number |
679c12e6 | 105 | ratio: number |
8ebf2a5d | 106 | allResolutions: number[] |
1ce4256a | 107 | hasAudio: boolean |
8ebf2a5d C |
108 | }) { |
109 | super() | |
110 | ||
111 | this.context = options.context | |
112 | this.user = options.user | |
113 | this.sessionId = options.sessionId | |
114 | this.videoLive = options.videoLive | |
115 | this.streamingPlaylist = options.streamingPlaylist | |
df1db951 | 116 | this.inputUrl = options.inputUrl |
8ebf2a5d | 117 | this.fps = options.fps |
679c12e6 | 118 | |
c826f34a | 119 | this.bitrate = options.bitrate |
41085b15 | 120 | this.ratio = options.ratio |
679c12e6 | 121 | |
1ce4256a C |
122 | this.hasAudio = options.hasAudio |
123 | ||
8ebf2a5d C |
124 | this.allResolutions = options.allResolutions |
125 | ||
126 | this.videoId = this.videoLive.Video.id | |
127 | this.videoUUID = this.videoLive.Video.uuid | |
128 | ||
129 | this.saveReplay = this.videoLive.saveReplay | |
130 | ||
4ec52d04 C |
131 | this.outDirectory = getLiveDirectory(this.videoLive.Video) |
132 | this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) | |
133 | ||
cfd57d2c C |
134 | this.liveSegmentShaStore = new LiveSegmentShaStore({ |
135 | videoUUID: this.videoLive.Video.uuid, | |
136 | sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename), | |
137 | streamingPlaylist: this.streamingPlaylist, | |
138 | sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED | |
139 | }) | |
140 | ||
8ebf2a5d C |
141 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) |
142 | } | |
143 | ||
144 | async runMuxing () { | |
145 | this.createFiles() | |
146 | ||
4ec52d04 | 147 | await this.prepareDirectories() |
8ebf2a5d C |
148 | |
149 | this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED | |
150 | ? await getLiveTranscodingCommand({ | |
df1db951 | 151 | inputUrl: this.inputUrl, |
764b1a14 | 152 | |
4ec52d04 | 153 | outPath: this.outDirectory, |
764b1a14 C |
154 | masterPlaylistName: this.streamingPlaylist.playlistFilename, |
155 | ||
f443a746 C |
156 | latencyMode: this.videoLive.latencyMode, |
157 | ||
8ebf2a5d C |
158 | resolutions: this.allResolutions, |
159 | fps: this.fps, | |
c826f34a | 160 | bitrate: this.bitrate, |
679c12e6 | 161 | ratio: this.ratio, |
c826f34a | 162 | |
1ce4256a C |
163 | hasAudio: this.hasAudio, |
164 | ||
8ebf2a5d C |
165 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
166 | profile: CONFIG.LIVE.TRANSCODING.PROFILE | |
167 | }) | |
f443a746 C |
168 | : getLiveMuxingCommand({ |
169 | inputUrl: this.inputUrl, | |
4ec52d04 | 170 | outPath: this.outDirectory, |
f443a746 C |
171 | masterPlaylistName: this.streamingPlaylist.playlistFilename, |
172 | latencyMode: this.videoLive.latencyMode | |
173 | }) | |
8ebf2a5d | 174 | |
4c6757f2 | 175 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) |
8ebf2a5d | 176 | |
b34ee7fa | 177 | this.watchMasterFile() |
cfd57d2c C |
178 | this.watchTSFiles() |
179 | this.watchM3U8File() | |
8ebf2a5d | 180 | |
4c6757f2 C |
181 | let ffmpegShellCommand: string |
182 | this.ffmpegCommand.on('start', cmdline => { | |
183 | ffmpegShellCommand = cmdline | |
184 | ||
185 | logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) | |
186 | }) | |
187 | ||
8ebf2a5d | 188 | this.ffmpegCommand.on('error', (err, stdout, stderr) => { |
b34ee7fa | 189 | this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) |
8ebf2a5d C |
190 | }) |
191 | ||
26e3e98f C |
192 | this.ffmpegCommand.on('end', () => { |
193 | this.emit('ffmpeg-end', ({ videoId: this.videoId })) | |
194 | ||
b34ee7fa | 195 | this.onFFmpegEnded() |
26e3e98f | 196 | }) |
8ebf2a5d C |
197 | |
198 | this.ffmpegCommand.run() | |
199 | } | |
200 | ||
201 | abort () { | |
609a4442 | 202 | if (!this.ffmpegCommand) return |
8ebf2a5d | 203 | |
39d117a4 | 204 | this.aborted = true |
8ebf2a5d | 205 | this.ffmpegCommand.kill('SIGINT') |
609a4442 C |
206 | } |
207 | ||
208 | destroy () { | |
209 | this.removeAllListeners() | |
210 | this.isAbleToUploadVideoWithCache.clear() | |
211 | this.hasClientSocketInBadHealthWithCache.clear() | |
8ebf2a5d C |
212 | } |
213 | ||
4c6757f2 C |
214 | private onFFmpegError (options: { |
215 | err: any | |
216 | stdout: string | |
217 | stderr: string | |
4c6757f2 C |
218 | ffmpegShellCommand: string |
219 | }) { | |
b34ee7fa | 220 | const { err, stdout, stderr, ffmpegShellCommand } = options |
4c6757f2 | 221 | |
b34ee7fa | 222 | this.onFFmpegEnded() |
8ebf2a5d C |
223 | |
224 | // Don't care that we killed the ffmpeg process | |
225 | if (err?.message?.includes('Exiting normally')) return | |
226 | ||
4c6757f2 | 227 | logger.error('Live transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) |
8ebf2a5d | 228 | |
26e3e98f | 229 | this.emit('ffmpeg-error', ({ videoId: this.videoId })) |
8ebf2a5d C |
230 | } |
231 | ||
b34ee7fa | 232 | private onFFmpegEnded () { |
4c6757f2 | 233 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags()) |
8ebf2a5d C |
234 | |
235 | setTimeout(() => { | |
236 | // Wait latest segments generation, and close watchers | |
237 | ||
cfd57d2c | 238 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ]) |
8ebf2a5d C |
239 | .then(() => { |
240 | // Process remaining segments hash | |
241 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | |
b34ee7fa | 242 | this.processSegments(this.segmentsToProcessPerPlaylist[key]) |
8ebf2a5d C |
243 | } |
244 | }) | |
245 | .catch(err => { | |
246 | logger.error( | |
b34ee7fa | 247 | 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, |
4c6757f2 | 248 | { err, ...this.lTags() } |
8ebf2a5d C |
249 | ) |
250 | }) | |
251 | ||
252 | this.emit('after-cleanup', { videoId: this.videoId }) | |
253 | }, 1000) | |
254 | } | |
255 | ||
b34ee7fa C |
256 | private watchMasterFile () { |
257 | this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) | |
8ebf2a5d | 258 | |
cfd57d2c C |
259 | this.masterWatcher.on('add', async () => { |
260 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | |
261 | try { | |
262 | const url = await storeHLSFileFromFilename(this.streamingPlaylist, this.streamingPlaylist.playlistFilename) | |
263 | ||
264 | this.streamingPlaylist.playlistUrl = url | |
265 | await this.streamingPlaylist.save() | |
266 | } catch (err) { | |
267 | logger.error('Cannot upload live master file to object storage.', { err, ...this.lTags() }) | |
268 | } | |
269 | } | |
270 | ||
271 | this.masterPlaylistCreated = true | |
8ebf2a5d C |
272 | |
273 | this.masterWatcher.close() | |
b34ee7fa | 274 | .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) |
8ebf2a5d C |
275 | }) |
276 | } | |
277 | ||
cfd57d2c C |
278 | private watchM3U8File () { |
279 | this.m3u8Watcher = watch(this.outDirectory + '/*.m3u8') | |
280 | ||
b3ce3606 C |
281 | const sendQueues = new Map<string, PQueue>() |
282 | ||
cfd57d2c C |
283 | const onChangeOrAdd = async (m3u8Path: string) => { |
284 | if (this.streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return | |
285 | ||
286 | try { | |
b3ce3606 C |
287 | if (!sendQueues.has(m3u8Path)) { |
288 | sendQueues.set(m3u8Path, new PQueue({ concurrency: 1 })) | |
289 | } | |
290 | ||
291 | const queue = sendQueues.get(m3u8Path) | |
292 | await queue.add(() => storeHLSFileFromPath(this.streamingPlaylist, m3u8Path)) | |
cfd57d2c C |
293 | } catch (err) { |
294 | logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() }) | |
295 | } | |
296 | } | |
297 | ||
298 | this.m3u8Watcher.on('change', onChangeOrAdd) | |
299 | } | |
300 | ||
b34ee7fa | 301 | private watchTSFiles () { |
8ebf2a5d C |
302 | const startStreamDateTime = new Date().getTime() |
303 | ||
b34ee7fa | 304 | this.tsWatcher = watch(this.outDirectory + '/*.ts') |
8ebf2a5d C |
305 | |
306 | const playlistIdMatcher = /^([\d+])-/ | |
307 | ||
75b7117f | 308 | const addHandler = async (segmentPath: string) => { |
4c6757f2 | 309 | logger.debug('Live add handler of %s.', segmentPath, this.lTags()) |
8ebf2a5d C |
310 | |
311 | const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] | |
312 | ||
313 | const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] | |
b34ee7fa | 314 | this.processSegments(segmentsToProcess) |
8ebf2a5d C |
315 | |
316 | this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] | |
317 | ||
318 | if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { | |
319 | this.emit('bad-socket-health', { videoId: this.videoId }) | |
320 | return | |
321 | } | |
322 | ||
323 | // Duration constraint check | |
324 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | |
325 | this.emit('duration-exceeded', { videoId: this.videoId }) | |
326 | return | |
327 | } | |
328 | ||
329 | // Check user quota if the user enabled replay saving | |
330 | if (await this.isQuotaExceeded(segmentPath) === true) { | |
331 | this.emit('quota-exceeded', { videoId: this.videoId }) | |
332 | } | |
333 | } | |
334 | ||
cfd57d2c C |
335 | const deleteHandler = async (segmentPath: string) => { |
336 | try { | |
337 | await this.liveSegmentShaStore.removeSegmentSha(segmentPath) | |
338 | } catch (err) { | |
339 | logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) | |
340 | } | |
341 | ||
342 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | |
343 | try { | |
aa887096 | 344 | await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath) |
cfd57d2c C |
345 | } catch (err) { |
346 | logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) | |
347 | } | |
348 | } | |
349 | } | |
8ebf2a5d C |
350 | |
351 | this.tsWatcher.on('add', p => addHandler(p)) | |
352 | this.tsWatcher.on('unlink', p => deleteHandler(p)) | |
353 | } | |
354 | ||
355 | private async isQuotaExceeded (segmentPath: string) { | |
356 | if (this.saveReplay !== true) return false | |
29399256 | 357 | if (this.aborted) return false |
8ebf2a5d C |
358 | |
359 | try { | |
360 | const segmentStat = await stat(segmentPath) | |
361 | ||
362 | LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.videoLive.id, segmentStat.size) | |
363 | ||
364 | const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) | |
365 | ||
366 | return canUpload !== true | |
367 | } catch (err) { | |
4c6757f2 | 368 | logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags() }) |
8ebf2a5d C |
369 | } |
370 | } | |
371 | ||
372 | private createFiles () { | |
373 | for (let i = 0; i < this.allResolutions.length; i++) { | |
374 | const resolution = this.allResolutions[i] | |
375 | ||
376 | const file = new VideoFileModel({ | |
377 | resolution, | |
378 | size: -1, | |
379 | extname: '.ts', | |
380 | infoHash: null, | |
381 | fps: this.fps, | |
cfd57d2c | 382 | storage: this.streamingPlaylist.storage, |
8ebf2a5d C |
383 | videoStreamingPlaylistId: this.streamingPlaylist.id |
384 | }) | |
385 | ||
386 | VideoFileModel.customUpsert(file, 'streaming-playlist', null) | |
4c6757f2 | 387 | .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags() })) |
8ebf2a5d C |
388 | } |
389 | } | |
390 | ||
391 | private async prepareDirectories () { | |
4ec52d04 | 392 | await ensureDir(this.outDirectory) |
8ebf2a5d C |
393 | |
394 | if (this.videoLive.saveReplay === true) { | |
4ec52d04 | 395 | await ensureDir(this.replayDirectory) |
8ebf2a5d | 396 | } |
8ebf2a5d C |
397 | } |
398 | ||
399 | private isDurationConstraintValid (streamingStartTime: number) { | |
400 | const maxDuration = CONFIG.LIVE.MAX_DURATION | |
401 | // No limit | |
402 | if (maxDuration < 0) return true | |
403 | ||
404 | const now = new Date().getTime() | |
405 | const max = streamingStartTime + maxDuration | |
406 | ||
407 | return now <= max | |
408 | } | |
409 | ||
b34ee7fa | 410 | private processSegments (segmentPaths: string[]) { |
cfd57d2c C |
411 | mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment)) |
412 | .catch(err => { | |
413 | if (this.aborted) return | |
414 | ||
415 | logger.error('Cannot process segments', { err, ...this.lTags() }) | |
416 | }) | |
417 | } | |
8ebf2a5d | 418 | |
cfd57d2c C |
419 | private async processSegment (segmentPath: string) { |
420 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | |
421 | await this.liveSegmentShaStore.addSegmentSha(segmentPath) | |
422 | ||
423 | if (this.saveReplay) { | |
424 | await this.addSegmentToReplay(segmentPath) | |
425 | } | |
426 | ||
427 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | |
428 | try { | |
429 | await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) | |
430 | } catch (err) { | |
431 | logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() }) | |
8ebf2a5d | 432 | } |
cfd57d2c | 433 | } |
29399256 | 434 | |
cfd57d2c C |
435 | // Master playlist and segment JSON file are created, live is ready |
436 | if (this.masterPlaylistCreated && !this.liveReady) { | |
437 | this.liveReady = true | |
438 | ||
439 | this.emit('live-ready', { videoId: this.videoId }) | |
440 | } | |
8ebf2a5d C |
441 | } |
442 | ||
443 | private hasClientSocketInBadHealth (sessionId: string) { | |
444 | const rtmpSession = this.context.sessions.get(sessionId) | |
445 | ||
446 | if (!rtmpSession) { | |
4c6757f2 | 447 | logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags()) |
8ebf2a5d C |
448 | return |
449 | } | |
450 | ||
451 | for (const playerSessionId of rtmpSession.players) { | |
452 | const playerSession = this.context.sessions.get(playerSessionId) | |
453 | ||
454 | if (!playerSession) { | |
4c6757f2 | 455 | logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags()) |
8ebf2a5d C |
456 | continue |
457 | } | |
458 | ||
459 | if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { | |
460 | return true | |
461 | } | |
462 | } | |
463 | ||
464 | return false | |
465 | } | |
466 | ||
b34ee7fa | 467 | private async addSegmentToReplay (segmentPath: string) { |
8ebf2a5d | 468 | const segmentName = basename(segmentPath) |
4ec52d04 | 469 | const dest = join(this.replayDirectory, buildConcatenatedName(segmentName)) |
8ebf2a5d C |
470 | |
471 | try { | |
472 | const data = await readFile(segmentPath) | |
473 | ||
474 | await appendFile(dest, data) | |
475 | } catch (err) { | |
4c6757f2 | 476 | logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags() }) |
8ebf2a5d C |
477 | } |
478 | } | |
479 | } | |
480 | ||
481 | // --------------------------------------------------------------------------- | |
482 | ||
483 | export { | |
484 | MuxingSession | |
485 | } |