Media: Video Segmentation Prototype (#12335)

Co-authored-by: Jonny Harris <spacedmonkey@users.noreply.github.com>
Co-authored-by: Pascal Birchler <pascalb@google.com>

includes/Experiments.php

@@ -299,6 +299,17 @@ public function get_experiments(): array {
 				'description' => __( 'Enable detailed page advancement settings on a per-page basis', 'web-stories' ),
 				'group'       => 'editor',
 			],
+			/**
+			 * Author: @timarney
+			 * Issue: #12164
+			 * Creation date: 2022-09-19
+			 */
+			[
+				'name'        => 'segmentVideo',
+				'label'       => __( 'Segment video', 'web-stories' ),
+				'description' => __( 'Enable support for segmenting video files', 'web-stories' ),
+				'group'       => 'editor',
+			],
 		];
 	}
 

packages/design-system/src/utils/panelTypes.js

@@ -29,6 +29,7 @@ const SHAPE_STYLE = 'shapeStyle';
 const TEXT_ACCESSIBILITY = 'textAccessibility';
 const TEXT_STYLE = 'textStyle';
 const VIDEO_OPTIONS = 'videoOptions';
+const VIDEO_SEGMENT = 'videoSegment';
 const VIDEO_ACCESSIBILITY = 'videoAccessibility';
 const ELEMENT_ALIGNMENT = 'elementAlignment';
 const PRODUCT = 'product';
@@ -47,6 +48,7 @@ const PanelTypes = {
   BORDER,
   ANIMATION,
   VIDEO_OPTIONS,
+  VIDEO_SEGMENT,
   CAPTIONS,
   LINK,
   IMAGE_ACCESSIBILITY,

packages/element-library/src/video/constants.js

@@ -63,6 +63,7 @@ export const panels = [
   PanelTypes.ELEMENT_ALIGNMENT,
   ...MEDIA_PANELS,
   PanelTypes.VIDEO_OPTIONS,
+  PanelTypes.VIDEO_SEGMENT,
   PanelTypes.VIDEO_ACCESSIBILITY,
   PanelTypes.CAPTIONS,
 ];

packages/story-editor/src/app/media/local/useContextValueProvider.js

@@ -145,6 +145,7 @@ export default function useContextValueProvider(reducerState, reducerActions) {
     isCurrentResourceMuting,
     isCurrentResourceTrimming,
     canTranscodeResource,
+    isBatchUploading,
   } = useUploadMedia({
     media,
     prependMedia,
@@ -284,6 +285,7 @@ export default function useContextValueProvider(reducerState, reducerActions) {
     muteExistingVideo,
     cropExistingVideo,
     trimExistingVideo,
+    segmentVideo,
   } = useProcessMedia({
     postProcessingResource,
     uploadMedia,
@@ -325,6 +327,7 @@ export default function useContextValueProvider(reducerState, reducerActions) {
       isCurrentResourceMuting,
       isCurrentResourceTrimming,
       canTranscodeResource,
+      isBatchUploading,
     },
     actions: {
       setNextPage,
@@ -344,6 +347,7 @@ export default function useContextValueProvider(reducerState, reducerActions) {
       updateBaseColor,
       updateBlurHash,
       cropExistingVideo,
+      segmentVideo,
     },
   };
 }

packages/story-editor/src/app/media/useUploadMedia.js

@@ -25,6 +25,7 @@ import {
   LOCAL_STORAGE_PREFIX,
 } from '@googleforcreators/design-system';
 import { isAnimatedGif } from '@googleforcreators/media';
+import { v4 as uuidv4 } from 'uuid';
 
 /**
  * Internal dependencies
@@ -77,6 +78,7 @@ function useUploadMedia({
       isCurrentResourceTranscoding,
       isCurrentResourceMuting,
       isCurrentResourceTrimming,
+      isBatchUploading,
       canTranscodeResource,
     },
     actions: { addItem, removeItem, finishItem },
@@ -155,6 +157,7 @@ function useUploadMedia({
       resource,
       onUploadSuccess,
       previousResourceId,
+      additionalData,
     } of uploaded) {
       const { id: resourceId } = resource;
       if (!resource) {
@@ -170,9 +173,17 @@ function useUploadMedia({
       // will cause things like base color and BlurHash generation to run
       // twice for a given resource.
       if (onUploadSuccess) {
-        onUploadSuccess({ id: resourceId, resource: resource });
+        onUploadSuccess({
+          id: resourceId,
+          resource: resource,
+          batchPosition: additionalData?.batchPosition,
+        });
         if (previousResourceId) {
-          onUploadSuccess({ id: previousResourceId, resource: resource });
+          onUploadSuccess({
+            id: previousResourceId,
+            resource: resource,
+            batchPosition: additionalData?.batchPosition,
+          });
         }
       }
 
@@ -250,7 +261,7 @@ function useUploadMedia({
      * @param {Blob} args.posterFile Blob object of poster.
      * @param {number} args.originalResourceId Original resource id.
      * @param {string} args.elementId ID of element on the canvas.
-     * @return {void}
+     * @return {string|null} Batch ID of the uploaded files on success, null otherwise.
      */
     async (
       files,
@@ -271,15 +282,16 @@ function useUploadMedia({
     ) => {
       // If there are no files passed, don't try to upload.
       if (!files?.length) {
-        return;
+        return null;
       }
 
+      const batchId = uuidv4();
+
       await Promise.all(
-        files.reverse().map(async (file) => {
+        files.reverse().map(async (file, index) => {
           // First, let's make sure the files we're trying to upload are actually valid.
           // We don't want to display placeholders / progress bars for items that
           // aren't supported anyway.
-
           const canTranscode = isTranscodingEnabled && canTranscodeFile(file);
           const isTooLarge = canTranscode && isFileTooLarge(file);
 
@@ -335,7 +347,11 @@ function useUploadMedia({
             onUploadProgress,
             onUploadError,
             onUploadSuccess,
-            additionalData,
+            additionalData: {
+              ...additionalData,
+              batchPosition: files.length - 1 - index,
+              batchId,
+            },
             posterFile,
             muteVideo,
             cropVideo,
@@ -346,6 +362,8 @@ function useUploadMedia({
           });
         })
       );
+
+      return batchId;
     },
     [
       showSnackbar,
@@ -371,6 +389,7 @@ function useUploadMedia({
     isCurrentResourceTranscoding,
     isCurrentResourceMuting,
     isCurrentResourceTrimming,
+    isBatchUploading,
     canTranscodeResource,
   };
 }

packages/story-editor/src/app/media/utils/test/useProcessMedia.js

@@ -27,6 +27,7 @@ import APIContext from '../../../api/context';
 import StoryContext from '../../../story/context';
 import useProcessMedia from '../useProcessMedia';
 import useMediaInfo from '../useMediaInfo';
+import { ConfigProvider } from '../../../config';
 
 const fetchRemoteFileMock = (url, mimeType) => {
   if (url === 'http://www.google.com/foo.mov') {
@@ -110,12 +111,19 @@ function setup() {
   const storyContextValue = {
     actions: { updateElementsByResourceId },
   };
+  const configState = {
+    capabilities: {
+      hasUploadMediaAction: true,
+    },
+  };
   const wrapper = ({ children }) => (
-    <APIContext.Provider value={apiContextValue}>
-      <StoryContext.Provider value={storyContextValue}>
-        {children}
-      </StoryContext.Provider>
-    </APIContext.Provider>
+    <ConfigProvider config={configState}>
+      <APIContext.Provider value={apiContextValue}>
+        <StoryContext.Provider value={storyContextValue}>
+          {children}
+        </StoryContext.Provider>
+      </APIContext.Provider>
+    </ConfigProvider>
   );
 
   const uploadVideoPoster = jest.fn();

packages/story-editor/src/app/media/utils/useFFmpeg.js

@@ -282,6 +282,81 @@ function useFFmpeg() {
     [getFFmpegInstance]
   );
 
+  /**
+   * Segment a video using FFmpeg.
+   *
+   * @param {File} file Original video file object.
+   * @param {string} segmentTime number of secs to split the video into.
+   * @return {Promise<File[]>} Segmented video files .
+   */
+  const segmentVideo = useCallback(
+    async (file, segmentTime, fileLength) => {
+      //eslint-disable-next-line @wordpress/no-unused-vars-before-return -- False positive because of the finally().
+      const trackTiming = getTimeTracker('segment_video');
+      let ffmpeg;
+      try {
+        ffmpeg = await getFFmpegInstance(file);
+        const type = file?.type || MEDIA_TRANSCODED_MIME_TYPE;
+        const ext = getExtensionFromMimeType(type);
+        const outputFileName = getFileBasename(file) + '_%03d.' + ext;
+        const keyframes = [];
+        for (let i = segmentTime; i < fileLength; i += segmentTime) {
+          keyframes.push(i);
+        }
+        const segmentTimes = keyframes.join(',');
+
+        await ffmpeg.run(
+          '-i',
+          file.name,
+          '-c',
+          'copy',
+          '-map',
+          '0',
+          '-force_key_frames',
+          `${segmentTimes}`,
+          '-f',
+          'segment',
+          '-segment_times',
+          `${segmentTimes}`,
+          '-segment_time_delta', //account for possible roundings operated when setting key frame times.
+          `${(1 / (2 * FFMPEG_CONFIG.FPS[1])).toFixed(2)}`,
+          '-reset_timestamps',
+          '1',
+          outputFileName
+        );
+
+        const files = [];
+        await ffmpeg
+          .FS('readdir', '/')
+          .filter(
+            (outputFile) =>
+              outputFile !== file.name && outputFile.endsWith(`.${ext}`)
+          )
+          .forEach(async (outputFile) => {
+            const data = await ffmpeg.FS('readFile', outputFile);
+            files.push(
+              blobToFile(new Blob([data.buffer], { type }), outputFile, type)
+            );
+          });
+        return files.sort((a, b) => a.name.localeCompare(b.name));
+      } catch (err) {
+        // eslint-disable-next-line no-console -- We want to surface this error.
+        console.error(err);
+        trackError('segment_video', err.message);
+        throw err;
+      } finally {
+        try {
+          ffmpeg.exit();
+        } catch {
+          // Not interested in errors here.
+        }
+
+        trackTiming();
+      }
+    },
+    [getFFmpegInstance]
+  );
+
   /**
    * Trim Video using FFmpeg.
    *
@@ -601,6 +676,7 @@ function useFFmpeg() {
       convertToMp3,
       trimVideo,
       cropVideo,
+      segmentVideo,
     }),
     [
       isTranscodingEnabled,
@@ -612,6 +688,7 @@ function useFFmpeg() {
       convertToMp3,
       trimVideo,
       cropVideo,
+      segmentVideo,
     ]
   );
 }

packages/story-editor/src/app/media/utils/useMediaInfo.js

@@ -149,6 +149,11 @@ function useMediaInfo() {
         // This will expose the window.MediaInfo global.
         await loadScriptOnce(mediainfoUrl);
 
+        // If for some reason it's not available yet.
+        if (!window.MediaInfo) {
+          return null;
+        }
+
         const mediaInfo = await window.MediaInfo({ format: 'JSON' });
         const result = JSON.parse(
           await mediaInfo.analyzeData(getSize, readChunk)

packages/story-editor/src/app/media/utils/useMediaUploadQueue/index.js

@@ -82,6 +82,7 @@ function useMediaUploadQueue() {
   const { isConsideredOptimized } = useMediaInfo();
 
   const [state, actions] = useReduction(initialState, reducer);
+
   const { uploadVideoPoster } = useUploadVideoFrame({
     updateMediaElement: noop,
   });
@@ -864,6 +865,20 @@ function useMediaUploadQueue() {
           item.state === ITEM_STATUS.MUTING && item.resource.id === resourceId
       );
 
+    /**
+     * Determine whether a batch of resources is being uploaded.
+     *
+     * batchId is available when uploading a new array of files
+     *
+     * @param {number} batchId Resource batchId.
+     * @return {boolean} Whether the batch of resources is uploading.
+     */
+    const isBatchUploading = (batchId) => {
+      return state.queue.some(
+        (item) => item.additionalData?.batchId === batchId
+      );
+    };
+
     /**
      * Determine whether the current resource is being trimmed.
      *
@@ -971,6 +986,7 @@ function useMediaUploadQueue() {
         isNewResourceProcessing,
         isNewResourceTranscoding,
         isElementTrimming,
+        isBatchUploading,
         canTranscodeResource,
       },
       actions: {

packages/story-editor/src/app/media/utils/useMediaUploadQueue/test/useMediaUploadQueue.js

@@ -148,6 +148,7 @@ describe('useMediaUploadQueue', () => {
         isCurrentResourceTrimming: expect.any(Function),
         isCurrentResourceUploading: expect.any(Function),
         canTranscodeResource: expect.any(Function),
+        isBatchUploading: expect.any(Function),
       })
     );
   });
@@ -365,6 +366,7 @@ describe('useMediaUploadQueue', () => {
         isCurrentResourceTrimming: expect.any(Function),
         isCurrentResourceUploading: expect.any(Function),
         canTranscodeResource: expect.any(Function),
+        isBatchUploading: expect.any(Function),
       })
     );
   });