From f44cb9694412d29889741a4aac4a2f51e00602ca Mon Sep 17 00:00:00 2001 From: mygittime <494841670@qq.com> Date: Sun, 20 Mar 2022 06:47:44 +0800 Subject: [PATCH 1/7] multithreading (#57) Co-authored-by: pengpeng.zhai --- cli.js | 1 + index.js | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/cli.js b/cli.js index f004f2d..b66a42c 100755 --- a/cli.js +++ b/cli.js @@ -43,6 +43,7 @@ commander .option('-O, --output ', 'Name of output (default: video.mp4)') .option('-R, --fps ', 'Frames per second to capture (default: 60)', parseFloat) .option('-d, --duration ', 'Duration of capture, in seconds (default: 5)', parseFloat) + .option('-T, --thread ', 'Number of threads to open the browser (default: 1)', parseFloat) .option('--frames ', 'Number of frames to capture', parseInt) .option('-S, --selector ', 'CSS Selector of item to capture') .option('-V, --viewport ', 'Viewport dimensions, in pixels (e.g. 800,600)', function (str) { diff --git a/index.js b/index.js index e574933..9d0c901 100644 --- a/index.js +++ b/index.js @@ -34,7 +34,10 @@ const timesnap = require('timesnap'); const path = require('path'); const fs = require('fs'); const spawn = require('child_process').spawn; +const cpus = require('os').cpus().length; const defaultFPS = 60; +const defaultThread = 1; +const defaultDuration = 5; const makeFileDirectoryIfNeeded = function (filepath) { var dir = path.parse(filepath).dir, ind, currDir; @@ -76,6 +79,7 @@ module.exports = async function (config) { var outputOptions = config.outputOptions || []; var frameDirectory = config.tempDir || config.frameDir; var fps; + var thread; var frameMode = config.frameCache || !config.pipeMode; var pipeMode = config.pipeMode; var processError; @@ -109,6 +113,9 @@ module.exports = async function (config) { fps = defaultFPS; } + thread = config.thread || defaultThread; + if(thread > cpus){thread = cpus}; + const log = function () { if (!config.quiet) { // eslint-disable-next-line no-console @@ -197,7 +204,35 @@ module.exports = async function (config) { var overallError; try { - await timesnap(timesnapConfig); + if(thread == 1){ + await timesnap(timesnapConfig); + }else{ + var progress = []; + + var totalFrame = config.frames || config.duration * fps || defaultDuration * fps; + var startFrame = 0; + while(thread >= 1){ + let _frame = Math.floor(totalFrame/thread--); + let frameStart = startFrame; + let frameEnd = frameStart + _frame; + let threadConfig = Object.assign({},timesnapConfig,{ + shouldSkipFrame:page=>{ + return page.frameCount <= frameStart || page.frameCount > frameEnd; + } + }) + + progress.push(new Promise(resolve => { + timesnap(threadConfig).then(()=>{ + resolve() + }) + })) + + startFrame = frameEnd; + totalFrame -= _frame; + } + + await Promise.all(progress); + } if (convertProcess) { convertProcess.stdin.end(); } From 75ee52811b4473a7897e391a91ac9e3357638eba Mon Sep 17 00:00:00 2001 From: Steve Tung <9443242+tungs@users.noreply.github.com> Date: Sat, 19 Mar 2022 15:55:42 -0700 Subject: [PATCH 2/7] code formatting and cleanup --- index.js | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/index.js b/index.js index 9d0c901..30e8a85 100644 --- a/index.js +++ b/index.js @@ -36,7 +36,7 @@ const fs = require('fs'); const spawn = require('child_process').spawn; const cpus = require('os').cpus().length; const defaultFPS = 60; -const defaultThread = 1; +const defaultThreads = 1; const defaultDuration = 5; const makeFileDirectoryIfNeeded = function (filepath) { @@ -79,7 +79,7 @@ module.exports = async function (config) { var outputOptions = config.outputOptions || []; var frameDirectory = config.tempDir || config.frameDir; var fps; - var thread; + var threads; var frameMode = config.frameCache || !config.pipeMode; var pipeMode = config.pipeMode; var processError; @@ -113,8 +113,10 @@ module.exports = async function (config) { fps = defaultFPS; } - thread = config.thread || defaultThread; - if(thread > cpus){thread = cpus}; + threads = config.threads || defaultThreads; + if (threads > cpus) { + threads = cpus; + } const log = function () { if (!config.quiet) { @@ -204,33 +206,25 @@ module.exports = async function (config) { var overallError; try { - if(thread == 1){ + if (threads === 1) { await timesnap(timesnapConfig); }else{ var progress = []; - - var totalFrame = config.frames || config.duration * fps || defaultDuration * fps; + var framesLeft = config.frames || config.duration * fps || defaultDuration * fps; var startFrame = 0; - while(thread >= 1){ - let _frame = Math.floor(totalFrame/thread--); + while (threads >= 1) { + let frameLength = Math.floor(framesLeft / threads--); let frameStart = startFrame; - let frameEnd = frameStart + _frame; - let threadConfig = Object.assign({},timesnapConfig,{ - shouldSkipFrame:page=>{ + let frameEnd = frameStart + frameLength; + let threadConfig = Object.assign({} , timesnapConfig, { + shouldSkipFrame(page) { return page.frameCount <= frameStart || page.frameCount > frameEnd; } - }) - - progress.push(new Promise(resolve => { - timesnap(threadConfig).then(()=>{ - resolve() - }) - })) - + }); + progress.push(timesnap(threadConfig)); startFrame = frameEnd; - totalFrame -= _frame; + framesLeft -= frameLength; } - await Promise.all(progress); } if (convertProcess) { From 5f409106f766153cb41a8ff2b838f5b3d85a2ebf Mon Sep 17 00:00:00 2001 From: Steve Tung <9443242+tungs@users.noreply.github.com> Date: Sat, 19 Mar 2022 16:00:08 -0700 Subject: [PATCH 3/7] Renamed `config.thread` to `config.threads` --- cli.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli.js b/cli.js index b66a42c..b3490b3 100755 --- a/cli.js +++ b/cli.js @@ -43,7 +43,7 @@ commander .option('-O, --output ', 'Name of output (default: video.mp4)') .option('-R, --fps ', 'Frames per second to capture (default: 60)', parseFloat) .option('-d, --duration ', 'Duration of capture, in seconds (default: 5)', parseFloat) - .option('-T, --thread ', 'Number of threads to open the browser (default: 1)', parseFloat) + .option('-T, --threads ', 'Number of threads to open (default: 1)', parseInt) .option('--frames ', 'Number of frames to capture', parseInt) .option('-S, --selector ', 'CSS Selector of item to capture') .option('-V, --viewport ', 'Viewport dimensions, in pixels (e.g. 800,600)', function (str) { @@ -121,4 +121,4 @@ if (errors.length && !commander.quiet) { }); } -recorder(commander); \ No newline at end of file +recorder(commander); From ceaa0edbf1c5db97a97a334353b3557092dfc230 Mon Sep 17 00:00:00 2001 From: user Date: Sat, 19 Mar 2022 16:19:49 -0700 Subject: [PATCH 4/7] minor cleanup --- index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 30e8a85..4c8c215 100644 --- a/index.js +++ b/index.js @@ -208,7 +208,7 @@ module.exports = async function (config) { try { if (threads === 1) { await timesnap(timesnapConfig); - }else{ + } else { var progress = []; var framesLeft = config.frames || config.duration * fps || defaultDuration * fps; var startFrame = 0; @@ -217,8 +217,9 @@ module.exports = async function (config) { let frameStart = startFrame; let frameEnd = frameStart + frameLength; let threadConfig = Object.assign({} , timesnapConfig, { - shouldSkipFrame(page) { - return page.frameCount <= frameStart || page.frameCount > frameEnd; + shouldSkipFrame({ frameCount }) { + // frameCount is 1 based + return frameCount <= frameStart || frameCount > frameEnd; } }); progress.push(timesnap(threadConfig)); From cf25ffbbabe9b523816b7d326cdc7c4eb6cfd278 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 3 Apr 2022 12:16:06 -0700 Subject: [PATCH 5/7] updates timesnap dependency --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c1f7293..6b12ca1 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "license": "BSD-3-Clause", "dependencies": { "commander": "^2.11.0", - "timesnap": "0.3.2" + "timesnap": "0.3.3" }, "devDependencies": { "eslint": "^7.32.0" From d1a02f231bd850e0df5517251ac465508ebbe2ff Mon Sep 17 00:00:00 2001 From: user Date: Sun, 3 Apr 2022 12:18:07 -0700 Subject: [PATCH 6/7] updates package-lock.json --- package-lock.json | 72 +++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/package-lock.json b/package-lock.json index bfb4307..7136b7f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -300,9 +300,9 @@ } }, "debug": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", - "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } @@ -617,9 +617,9 @@ } }, "globals": { - "version": "13.12.1", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.1.tgz", - "integrity": "sha512-317dFlgY2pdJZ9rspXDks7073GpDmXdfbM3vYYp0HAMKGDh1FfWPleI2ljVNLQX5M5lXcAslTcPTrOrMEFOjyw==", + "version": "13.13.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.13.0.tgz", + "integrity": "sha512-EQ7Q18AJlPwp3vUDL4mKA0KXrXyNIQyWon6T6XQiBQF0XHvRsiCSrWmmeATpUzdJN2HhWZU6Pdl0a9zdep5p6A==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -773,37 +773,37 @@ "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==" }, "mime-db": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", - "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { - "version": "2.1.34", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", - "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { - "mime-db": "1.51.0" + "mime-db": "1.52.0" } }, "minimatch": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz", - "integrity": "sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "mkdirp": { - "version": "0.5.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", - "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "requires": { - "minimist": "^1.2.5" + "minimist": "^1.2.6" } }, "ms": { @@ -1076,9 +1076,9 @@ }, "dependencies": { "ajv": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.10.0.tgz", - "integrity": "sha512-bzqAEZOjkrUMl2afH8dknrq5KEk2SrwdBROR+vH1EKVQTqaUbJVPdc/gEdggTMM0Se+s+Ja4ju4TlNcStKl2Hw==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", @@ -1102,20 +1102,21 @@ "dev": true }, "timesnap": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/timesnap/-/timesnap-0.3.2.tgz", - "integrity": "sha512-6b2E7/KN/q3ZmOgH0EZCPy8lUpi+8MfVgVW3iy97bNPfdq4vtn3cTbhWGvxQZlp6y12mkeyxlSwNYQN4+e9RdA==", + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/timesnap/-/timesnap-0.3.3.tgz", + "integrity": "sha512-Y+KqywtlXbY/RbvfLRm9MofCnISoGpDj4OZJn8NT7IgAIqnVQ8ujKYcdNG7yEYQZUg/5eAx0av1IlWYLrct6xw==", "requires": { "commander": "^2.11.0", "puppeteer": "^2.1.1", "sprintf-js": "1.1.1", - "timeweb": "^0.2.1" + "timeweb": "0.3.1", + "unrandomize": "0.1.0" } }, "timeweb": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/timeweb/-/timeweb-0.2.2.tgz", - "integrity": "sha512-NvmA72+NNPKqzYwctCDQBMuSHYVkKhKzYdBzY9M2dD0ck8z44mMZYIHJbPcXh4bR4l49IyVdHawIyiGpPEFL/w==" + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/timeweb/-/timeweb-0.3.1.tgz", + "integrity": "sha512-3ZaDlwv6AecqoJ6QPRoOxdpWf9TwPbHxIxU0uCYVi5Re0JZ/yQ41C5CqAuDmcYvbKkGRngG29CPzAWlae9s8pw==" }, "type-check": { "version": "0.4.0", @@ -1137,6 +1138,11 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "unrandomize": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/unrandomize/-/unrandomize-0.1.0.tgz", + "integrity": "sha512-ps3jw749uVwC0RrRS193HU1pAqQeoZMQtKcwBabmZ4o14/dBC8KXoqR/tPX6Yg79d8CUcb4uI/IopVphhrKp3w==" + }, "uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", From d78c3310a27eca838d75a0370499da03f15a5098 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 3 Apr 2022 12:46:07 -0700 Subject: [PATCH 7/7] version bump to v0.3.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7136b7f..0eaa41e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "timecut", - "version": "0.3.3-prerelease", + "version": "0.3.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 6b12ca1..5dacb32 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "timecut", - "version": "0.3.3-prerelease", + "version": "0.3.3", "description": "Record smooth movies of web pages", "repository": { "type": "git",