Update import script to support subtitles

packages/core/src/Regexes.ts

@@ -1,5 +1,6 @@
 export const Regexes = {
-	VideoFile: /([\w\s':.-]+)(?: \((\d{4})\))?.mpd$/,
+	VideoFile: (ext = 'mpd') => new RegExp(`^([\\w\\s':\\.\\-!]+)(?: \\((\\d{4})\\))?\\.${ext}$`),
 	Thumbnail: /thumb\d*\.(jpg|jpeg|png)$/,
 	Poster: /poster\d*\.(jpg|jpeg|png)$/,
+	LangCheck: /^((?:(en-GB-oed|i-ami|i-bnn|i-default|i-enochian|i-hak|i-klingon|i-lux|i-mingo|i-navajo|i-pwn|i-tao|i-tay|i-tsu|sgn-BE-FR|sgn-BE-NL|sgn-CH-DE)|(art-lojban|cel-gaulish|no-bok|no-nyn|zh-guoyu|zh-hakka|zh-min|zh-min-nan|zh-xiang))|((?:([A-Za-z]{2,3}(-(?:[A-Za-z]{3}(-[A-Za-z]{3}){0,2}))?)|[A-Za-z]{4}|[A-Za-z]{5,8})(-(?:[A-Za-z]{4}))?(-(?:[A-Za-z]{2}|[0-9]{3}))?(-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(-(?:[0-9A-WY-Za-wy-z](-[A-Za-z0-9]{2,8})+))*(-(?:x(-[A-Za-z0-9]{1,8})+))?)|(?:x(-[A-Za-z0-9]{1,8})+))$/i // Source: https://www.regextester.com/103066
 };

packages/core/src/Services/Indexer/MovieIndexFetcher.ts

@@ -24,11 +24,11 @@ export class MovieIndexFetcher implements IIndexFetcher<IMovieMetaData[]> {
 
 	public async extractMovieMetaData(node: IIpfsService, entry: IFileInfo, skeleton?: any): Promise<IMovieMetaData> {
 		const files = (await this.node.ls(entry.cid)).filter(f => f.type == 'file');
-		const videoFile = files.find(f => Regexes.VideoFile.exec(f.name) != null);
+		const videoFile = files.find(f => Regexes.VideoFile('mpd').exec(f.name) != null);
 
 		if (!videoFile) throw new Error('Failed to find video file in ' + entry.name + '|' + entry.cid);
 
-		const videoData = Regexes.VideoFile.exec(videoFile.name)!;
+		const videoData = Regexes.VideoFile('mpd').exec(videoFile.name)!;
 
 		return {
 			...entry,

packages/core/tests/Regexes.test.ts

@@ -3,31 +3,31 @@ import { Regexes } from '../src';
 
 describe('Regexes', () => {
 	test('Video file gets matches', () => {
-		const res1 = Regexes.VideoFile.exec('Sample Movie (2015).mpd');
+		const res1 = Regexes.VideoFile('mpd').exec('Sample Movie (2015).mpd');
 		expect(res1).not.toBeNull();
 		expect(res1[0]).toEqual('Sample Movie (2015).mpd');
 		expect(res1[1]).toEqual('Sample Movie');
 		expect(res1[2]).toBe('2015');
 
-		const res2 = Regexes.VideoFile.exec('Sample Movie.mpd');
+		const res2 = Regexes.VideoFile('mpd').exec('Sample Movie.mpd');
 		expect(res2).not.toBeNull();
 		expect(res2[0]).toEqual('Sample Movie.mpd');
 		expect(res2[1]).toEqual('Sample Movie');
 		expect(res2[2]).toBe(undefined);
 
-		const res3 = Regexes.VideoFile.exec('Sample Movie: Subtitle (2015).mpd');
+		const res3 = Regexes.VideoFile('mpd').exec('Sample Movie: Subtitle (2015).mpd');
 		expect(res3).not.toBeNull();
 		expect(res3[0]).toEqual('Sample Movie: Subtitle (2015).mpd');
 		expect(res3[1]).toEqual('Sample Movie: Subtitle');
 		expect(res3[2]).toBe('2015');
 
-		const res4 = Regexes.VideoFile.exec('Sample\'s Movie Vol.2 (2015).mpd');
+		const res4 = Regexes.VideoFile('mpd').exec('Sample\'s Movie! Vol.2 (2015).mpd');
 		expect(res4).not.toBeNull();
-		expect(res4[0]).toEqual('Sample\'s Movie Vol.2 (2015).mpd');
-		expect(res4[1]).toEqual('Sample\'s Movie Vol.2');
+		expect(res4[0]).toEqual('Sample\'s Movie! Vol.2 (2015).mpd');
+		expect(res4[1]).toEqual('Sample\'s Movie! Vol.2');
 		expect(res4[2]).toBe('2015');
 
-		const res5 = Regexes.VideoFile.exec('Sample Movie - Subtitle (2015).mpd');
+		const res5 = Regexes.VideoFile('mpd').exec('Sample Movie - Subtitle (2015).mpd');
 		expect(res5).not.toBeNull();
 		expect(res5[0]).toEqual('Sample Movie - Subtitle (2015).mpd');
 		expect(res5[1]).toEqual('Sample Movie - Subtitle');
@@ -47,4 +47,23 @@ describe('Regexes', () => {
 		expect(Regexes.Poster.exec('Sample Movie (2015)-poster.jpg')).not.toBeNull();
 		expect(Regexes.Poster.exec('Sample Movie (2015)-poster.jpeg')).not.toBeNull();
 	});
+
+	test('Language check is valid', () => {
+		expect(Regexes.LangCheck.exec('de')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('i-enochian')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('zh-Hant')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('zh-cmn-Hans-CN')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('zh-Hans-CN')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('sl-rozaj')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('sl-rozaj-biske')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('de-CH-1901')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('sl-IT-nedis')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('de-DE')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('es-419')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('de-CH-x-phonebk')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('az-Arab-x-AZE-derbend')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('x-whatever')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('qaa-Qaaa-QM-x-southern')).not.toBeNull();
+		expect(Regexes.LangCheck.exec('de-Qaaa')).not.toBeNull();
+	});
 });

packages/tools/package.json

@@ -14,6 +14,7 @@
 		"fluent-ffmpeg": "^2.1.2",
 		"ipmc-core": "workspace:^",
 		"kubo-rpc-client": "^4.1.1",
+		"srt2vtt": "^1.3.1",
 		"yargs": "^17.7.2"
 	},
 	"devDependencies": {

packages/tools/src/import.ts

@@ -5,6 +5,10 @@ import { exec } from 'child_process';
 import { input } from '@inquirer/prompts';
 import { ITempDir, tempDir } from './utils/tempDIr';
 import chalk from 'chalk';
+import { Regexes } from 'ipmc-core';
+import path, { basename } from 'path';
+import srt2vtt from 'srt2vtt';
+import fs from 'fs';
 
 type TStream = 'Video' | 'Audio';
 
@@ -22,7 +26,7 @@ async function detectStreams(packager: string, file: string): Promise<IStream[]>
 				console.error(stderr);
 				reject(error);
 			} else {
-				console.log(stdout);
+				//console.debug(stdout);
 				const regex = /Stream \[(\d+)\] type: (\w+)\r?\n((?: [\w_]+: .+\r?\n)+)\r?\n/gm;
 				const matches = stdout.matchAll(regex);
 				const streams: IStream[] = [];
@@ -86,44 +90,80 @@ const args = yargs(hideBin(process.argv))
 		describe: 'ipfs api url',
 		default: 'http://127.0.0.1:5002/api/v0'
 	})
-	.option('file', {
-		alias: 'f',
-	})
-	.option('title', {
-		alias: 't',
-	})
-	.demandOption(['file', 'title'])
+	.array('file')
+	.alias('file', 'f')
+	.demandOption(['file'])
 	.help()
 	.parseSync();
 
 (async () => {
-	const streams = await detectStreams(args.packager, args.file as string);
-	console.log('Streams detected!');
+	const files = args.file as string[];
+
+	const temp = tempDir();
+	const outDir = tempDir();
+
+	try {
+
+		const streams: IStream[] = [];
+		for (const file of files) {
+			let actualFile = file;
+			if (file.endsWith('.srt')) {
+				const srtData = fs.readFileSync(file);
+				actualFile = await new Promise((resolve, reject) => {
+					srt2vtt(srtData, (err, vttData) => {
+						if (err) {
+							reject(err);
+						} else {
+							const fn = path.join(temp.getPath(), basename(file, 'srt') + 'vtt');
+							fs.writeFileSync(fn, vttData);
+							resolve(fn);
+						}
+					});
+				});
+			}
 
-	for (const stream of streams) {
-		if (stream.type !== 'Video') {
-			stream.lang = await input({ message: `[${stream.id}|${stream.file}] Language?`, default: stream.lang });
+			const streamsFromFile = await detectStreams(args.packager, actualFile);
+			streams.push(...streamsFromFile);
 		}
-	}
 
-	const dir = tempDir();
+		console.log(`Detected ${streams.length} streams, from ${files.length} files!`);
+
+		const movieData = files.map(file => Regexes.VideoFile('mp4').exec(path.basename(file))).find(r => r != null);
 
-	await packageStreams(args.packager, args.title as string, streams, dir);
-	console.log('Streams packaged!');
+		const title = await input({ message: 'Movie title?', default: movieData != null ? movieData[1] : undefined, required: true });
+		const year = await input({ message: 'Year?', default: movieData != null ? movieData[2] : undefined, required: true });
 
+		for (const stream of streams) {
+			const streamDisplayName = `${stream.id}|${stream.type}|${path.basename(stream.file)}`;
 
-	const node = create({ url: args.ipfs });
-	for await (const file of node.addAll(globSource(dir.getPath(), '**'), {
-		pin: false,
-		wrapWithDirectory: true
-	})) {
-		if (file.path === '') {
-			console.log(`Added to ipfs ${chalk.green(file.cid)}`);
+			if (stream.type !== 'Video') {
+				stream.lang = await input({
+					message: `[${streamDisplayName}] Language?`,
+					default: stream.lang,
+					required: true,
+					validate: (input) => Regexes.LangCheck.exec(input) != null
+				});
+			}
 		}
-	}
 
-	dir.clean();
+		console.log('Packaging...');
+		await packageStreams(args.packager, `${title} (${year})`, streams, outDir);
+		console.log('Streams packaged!');
+
+		const node = create({ url: args.ipfs });
+		for await (const file of node.addAll(globSource(outDir.getPath(), '**'), {
+			pin: false,
+			wrapWithDirectory: true
+		})) {
+			if (file.path === '') {
+				console.log(`Added to ipfs ${chalk.green(file.cid)}`);
+			}
+		}
 
-	console.log('done');
+		console.log('Done!');
+	} finally {
+		outDir.clean();
+		temp.clean();
+	}
 })();
 

packages/tools/src/utils/tempDIr.ts

@@ -6,7 +6,7 @@ export interface ITempDir {
 }
 
 export function tempDir(): ITempDir {
-	const id = 'test';
+	const id = crypto.randomUUID();
 	const path = `./tmp/${id}`;
 
 	function getPath(filename?: string): string {

yarn.lock

@@ -4822,6 +4822,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"codepage@npm:~1.4.0":
+  version: 1.4.0
+  resolution: "codepage@npm:1.4.0"
+  dependencies:
+    commander: "npm:"
+    concat-stream: "npm:"
+    exit-on-epipe: "npm:"
+    voc: "npm:"
+  bin:
+    codepage: ./bin/codepage.njs
+  checksum: 10c0/86820edde690e5fae00ee0e4b0a85cec604c153e963810a59ef59a49f11aeb874c8bac4b92ef4076159fc264e1041be6665e0852404df2637e6e1cf49a3bb6db
+  languageName: node
+  linkType: hard
+
 "color-convert@npm:^1.9.0":
   version: 1.9.3
   resolution: "color-convert@npm:1.9.3"
@@ -4863,6 +4877,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"commander@npm:":
+  version: 12.1.0
+  resolution: "commander@npm:12.1.0"
+  checksum: 10c0/6e1996680c083b3b897bfc1cfe1c58dfbcd9842fd43e1aaf8a795fbc237f65efcc860a3ef457b318e73f29a4f4a28f6403c3d653d021d960e4632dd45bde54a9
+  languageName: node
+  linkType: hard
+
 "commander@npm:^2.20.0":
   version: 2.20.3
   resolution: "commander@npm:2.20.3"
@@ -4919,6 +4940,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"concat-stream@npm:":
+  version: 2.0.0
+  resolution: "concat-stream@npm:2.0.0"
+  dependencies:
+    buffer-from: "npm:^1.0.0"
+    inherits: "npm:^2.0.3"
+    readable-stream: "npm:^3.0.2"
+    typedarray: "npm:^0.0.6"
+  checksum: 10c0/29565dd9198fe1d8cf57f6cc71527dbc6ad67e12e4ac9401feb389c53042b2dceedf47034cbe702dfc4fd8df3ae7e6bfeeebe732cc4fa2674e484c13f04c219a
+  languageName: node
+  linkType: hard
+
 "config-file-ts@npm:^0.2.4":
   version: 0.2.6
   resolution: "config-file-ts@npm:0.2.6"
@@ -6314,6 +6347,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"exit-on-epipe@npm:":
+  version: 1.0.1
+  resolution: "exit-on-epipe@npm:1.0.1"
+  checksum: 10c0/f10a5fbf1abb6294b06220f99d84bb918286700e8aec3d364963767f1f0530b7e5abf29d8f0ef2672458e794f746f73254d397b1596acc745bdce81586b183c0
+  languageName: node
+  linkType: hard
+
 "expand-template@npm:^2.0.3":
   version: 2.0.3
   resolution: "expand-template@npm:2.0.3"
@@ -7571,6 +7611,7 @@ __metadata:
     fluent-ffmpeg: "npm:^2.1.2"
     ipmc-core: "workspace:^"
     kubo-rpc-client: "npm:^4.1.1"
+    srt2vtt: "npm:^1.3.1"
     vite-node: "npm:^2.1.8"
     yargs: "npm:^17.7.2"
   languageName: unknown
@@ -10428,7 +10469,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0, readable-stream@npm:^3.6.0":
+"readable-stream@npm:^3.0.2, readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0, readable-stream@npm:^3.6.0":
   version: 3.6.2
   resolution: "readable-stream@npm:3.6.2"
   dependencies:
@@ -11100,6 +11141,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"srt2vtt@npm:^1.3.1":
+  version: 1.3.1
+  resolution: "srt2vtt@npm:1.3.1"
+  dependencies:
+    codepage: "npm:~1.4.0"
+    utfx: "npm:~1.0.0"
+  checksum: 10c0/a7f5be4d4ddf028d432cc3cd3269901d0d7509fcebf6c1c8b44c6ccfb8586089c4e53c0a64081af3abca00c893f40d271030c32ca2bb89569d5a1e1922448801
+  languageName: node
+  linkType: hard
+
 "ssri@npm:^10.0.0":
   version: 10.0.6
   resolution: "ssri@npm:10.0.6"
@@ -11822,6 +11873,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"typedarray@npm:^0.0.6":
+  version: 0.0.6
+  resolution: "typedarray@npm:0.0.6"
+  checksum: 10c0/6005cb31df50eef8b1f3c780eb71a17925f3038a100d82f9406ac2ad1de5eb59f8e6decbdc145b3a1f8e5836e17b0c0002fb698b9fe2516b8f9f9ff602d36412
+  languageName: node
+  linkType: hard
+
 "typescript@npm:5.4.2":
   version: 5.4.2
   resolution: "typescript@npm:5.4.2"
@@ -12022,6 +12080,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"utfx@npm:~1.0.0":
+  version: 1.0.1
+  resolution: "utfx@npm:1.0.1"
+  checksum: 10c0/5021c67018bd9e42272fd91681e16c900bec26015aa01ccdbc6b9cb83b96d8a978633628ad7be41535c362e0ce835afef4045cc45892e8a7052339a3b46f3452
+  languageName: node
+  linkType: hard
+
 "util-deprecate@npm:^1.0.1, util-deprecate@npm:~1.0.1":
   version: 1.0.2
   resolution: "util-deprecate@npm:1.0.2"
@@ -12293,6 +12358,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"voc@npm:":
+  version: 1.2.0
+  resolution: "voc@npm:1.2.0"
+  bin:
+    voc: voc.njs
+  checksum: 10c0/396eb455f5d239110afb24462afa73cd4c2b4ddaa3cb2ff1f7a22d08a5ddc32317c312b3639b9d77ab78a756da8a8efe7bb5dd0b9889815fd1456d44b4706b07
+  languageName: node
+  linkType: hard
+
 "void-elements@npm:3.1.0":
   version: 3.1.0
   resolution: "void-elements@npm:3.1.0"