[ts-sdk] Make react sdk run on browser

ts/@live-compositor/browser-render/src/index.ts

@@ -1,2 +1,3 @@
 export { loadWasmModule } from './wasm';
 export * from './renderer';
+export * from './api';

ts/@live-compositor/browser-render/src/renderer.ts

@@ -8,11 +8,6 @@ export type RendererOptions = {
   streamFallbackTimeoutMs: number;
 };
 
-export type Framerate = {
-  num: number;
-  den: number;
-};
-
 export type FrameSet = {
   ptsMs: number;
   frames: { [id: string]: Frame };

ts/@live-compositor/core/src/api.ts

@@ -1,5 +1,7 @@
 import { Api } from 'live-compositor';
 import { CompositorManager } from './compositorManager';
+import { RegisterOutputRequest } from './api/output';
+import { RegisterInputRequest } from './api/input';
 
 export { Api };
 
@@ -24,7 +26,7 @@ export class ApiClient {
     });
   }
 
-  public async registerOutput(outptuId: string, request: Api.RegisterOutput): Promise<object> {
+  public async registerOutput(outptuId: string, request: RegisterOutputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/output/${encodeURIComponent(outptuId)}/register`,
@@ -40,7 +42,7 @@ export class ApiClient {
     });
   }
 
-  public async registerInput(inputId: string, request: Api.RegisterInput): Promise<object> {
+  public async registerInput(inputId: string, request: RegisterInputRequest): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/input/${encodeURIComponent(inputId)}/register`,

ts/@live-compositor/core/src/api/input.ts

@@ -1,17 +1,27 @@
 import { Api } from '../api';
 import { RegisterInput, Inputs } from 'live-compositor';
 
-export function intoRegisterInput(input: RegisterInput): Api.RegisterInput {
-  if (input.type === 'mp4') {
-    return intoMp4RegisterInput(input);
-  } else if (input.type === 'rtp_stream') {
-    return intoRtpRegisterInput(input);
-  } else {
-    throw new Error(`Unknown input type ${(input as any).type}`);
+export type RegisterInputRequest = Api.RegisterInput | RegisterBytesInput;
+
+export type RegisterBytesInput = {
+  type: 'bytes';
+  video: Inputs.InputBytesVideoOptions;
+};
+
+export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
+  switch (input.type) {
+    case 'mp4':
+      return intoMp4RegisterInput(input);
+    case 'rtp_stream':
+      return intoRtpRegisterInput(input);
+    case 'bytes':
+      return intoBytesRegisterInput(input);
+    default:
+      throw new Error(`Unknown input type ${(input as any).type}`);
   }
 }
 
-function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): Api.RegisterInput {
+function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): RegisterInputRequest {
   return {
     type: 'mp4',
     url: input.url,
@@ -21,7 +31,7 @@ function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): Api.RegisterInput
   };
 }
 
-function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): Api.RegisterInput {
+function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): RegisterInputRequest {
   return {
     type: 'rtp_stream',
     port: input.port,
@@ -33,6 +43,13 @@ function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): Api.RegisterInput
   };
 }
 
+function intoBytesRegisterInput(input: Inputs.RegisterBytesInput): RegisterInputRequest {
+  return {
+    type: 'bytes',
+    video: input.video,
+  };
+}
+
 function intoInputAudio(audio: Inputs.InputRtpAudioOptions): Api.InputRtpAudioOptions {
   if (audio.decoder === 'opus') {
     return {

ts/@live-compositor/core/src/api/output.ts

@@ -1,22 +1,38 @@
-import { RegisterOutput, Api, Outputs } from 'live-compositor';
+import { RegisterOutput, Api, Outputs, OutputBytesFrameFormat } from 'live-compositor';
+
+export type RegisterOutputRequest = Api.RegisterOutput | RegisterBytesOutput;
+
+export type RegisterBytesOutput = {
+  type: 'bytes';
+  video?: OutputBytesVideoOptions;
+};
+
+export type OutputBytesVideoOptions = {
+  format: OutputBytesFrameFormat;
+  resolution: Api.Resolution;
+  initial: Api.Video;
+};
 
 export function intoRegisterOutput(
   output: RegisterOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
-  if (output.type === 'rtp_stream') {
-    return intoRegisterRtpOutput(output, initial);
-  } else if (output.type === 'mp4') {
-    return intoRegisterMp4Output(output, initial);
-  } else {
-    throw new Error(`Unknown input type ${(output as any).type}`);
+): RegisterOutputRequest {
+  switch (output.type) {
+    case 'rtp_stream':
+      return intoRegisterRtpOutput(output, initial);
+    case 'mp4':
+      return intoRegisterMp4Output(output, initial);
+    case 'bytes':
+      return intoRegisterBytesOutput(output, initial);
+    default:
+      throw new Error(`Unknown output type ${(output as any).type}`);
   }
 }
 
 function intoRegisterRtpOutput(
   output: Outputs.RegisterRtpOutput,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'rtp_stream',
     port: output.port,
@@ -30,7 +46,7 @@ function intoRegisterRtpOutput(
 function intoRegisterMp4Output(
   output: Outputs.RegisterMp4Output,
   initial: { video?: Api.Video; audio?: Api.Audio }
-): Api.RegisterOutput {
+): RegisterOutputRequest {
   return {
     type: 'mp4',
     path: output.serverPath,
@@ -39,6 +55,20 @@ function intoRegisterMp4Output(
   };
 }
 
+function intoRegisterBytesOutput(
+  output: Outputs.RegisterBytesOutput,
+  initial: { video?: Api.Video; _audio?: Api.Audio }
+): RegisterOutputRequest {
+  return {
+    type: 'bytes',
+    video: {
+      format: output.video.format,
+      resolution: output.video.resolution,
+      initial: initial.video!,
+    },
+  };
+}
+
 function intoOutputVideoOptions(
   video: Outputs.OutputRtpVideoOptions | Outputs.OutputMp4VideoOptions,
   initial: Api.Video

ts/@live-compositor/core/src/index.ts

@@ -1,3 +1,5 @@
 export { ApiClient, ApiRequest } from './api';
 export { LiveCompositor } from './compositor';
 export { CompositorManager } from './compositorManager';
+export { RegisterInputRequest, RegisterBytesInput } from './api/input';
+export { RegisterOutputRequest, RegisterBytesOutput } from './api/output';

ts/@live-compositor/core/src/output.ts

@@ -33,12 +33,13 @@ class Output {
       this.shouldUpdateWhenReady = true;
     };
 
-    if (registerRequest.audio) {
-      this.initialAudioConfig = registerRequest.audio.initial ?? { inputs: [] };
+    const hasAudio = 'audio' in registerRequest && !!registerRequest.audio;
+    if (hasAudio) {
+      this.initialAudioConfig = registerRequest.audio!.initial ?? { inputs: [] };
     }
 
     const onUpdate = () => this.throttledUpdate();
-    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, !!registerRequest.audio);
+    this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, hasAudio);
 
     if (registerRequest.video) {
       const rootElement = React.createElement(OutputRootComponent, {

ts/@live-compositor/core/tsconfig.json

@@ -1,6 +1,9 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ESNext"
   }
 }

ts/@live-compositor/node/tsconfig.json

@@ -1,6 +1,9 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ESNext"
   }
 }

ts/@live-compositor/web/.eslintignore

@@ -0,0 +1 @@
+dist

ts/@live-compositor/web/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "../../.eslintrc.base.json"
+  ]
+}

ts/@live-compositor/web/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@live-compositor/web",
+  "version": "0.1.0-rc.0",
+  "description": "",
+  "main": "dist/index.js",
+  "scripts": {
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit",
+    "watch": "tsc --watch --preserveWatchOutput",
+    "build": "tsc",
+    "clean": "rimraf dist",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "author": "",
+  "license": "MIT",
+  "files": [
+    "/dist"
+  ],
+  "dependencies": {
+    "@live-compositor/browser-render": "file:../browser-render",
+    "@live-compositor/core": "0.1.0-rc.3",
+    "live-compositor": "^0.1.0-rc.3"
+  }
+}

ts/@live-compositor/web/src/compositor.ts

@@ -0,0 +1,61 @@
+import { ImageSpec, InputId, OutputId, Renderer } from '@live-compositor/browser-render';
+import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core';
+import WasmInstance from './manager/wasmInstance';
+import { RegisterInput, RegisterOutput } from 'live-compositor';
+import { Queue } from './queue';
+import Input from './input';
+import Output from './output';
+
+export type LiveCompositorOptions = {
+  framerate: Framerate;
+  streamFallbackTimeoutMs: number;
+};
+
+export type Framerate = {
+  num: number;
+  den: number;
+};
+
+export default class LiveCompositor {
+  private compositor: CoreLiveCompositor;
+  private renderer: Renderer;
+  private framerate: Framerate;
+  private inputs: Map<InputId, Input>;
+  private outputs: Map<OutputId, Output>;
+
+  private constructor(compositor: CoreLiveCompositor, renderer: Renderer, framerate: Framerate) {
+    this.compositor = compositor;
+    this.renderer = renderer;
+    this.framerate = framerate;
+    this.inputs = new Map();
+    this.outputs = new Map();
+  }
+
+  // TODO(noituri): Refactor this
+  public static async create(options: LiveCompositorOptions): Promise<LiveCompositor> {
+    const renderer = await Renderer.create({
+      streamFallbackTimeoutMs: options.streamFallbackTimeoutMs,
+    });
+    const compositor = new CoreLiveCompositor(new WasmInstance(renderer));
+    await compositor.init();
+    return new LiveCompositor(compositor, renderer, options.framerate);
+  }
+
+  // TODO(noituri): Add missing endpoits
+
+  public async registerOutput(outputId: string, request: RegisterOutput): Promise<object> {
+    return this.compositor.registerOutput(outputId, request);
+  }
+
+  public async registerInput(inputId: string, request: RegisterInput): Promise<object> {
+    return this.compositor.registerInput(inputId, request);
+  }
+
+  public async registerImage(imageId: string, request: ImageSpec): Promise<object> {
+    return this.compositor.registerImage(imageId, request);
+  }
+
+  public async start(): Promise<void> {
+    Queue.start(this.framerate, this.inputs, this.outputs, this.renderer);
+  }
+}

ts/@live-compositor/web/src/index.ts

@@ -0,0 +1,4 @@
+import WasmInstance from './manager/wasmInstance';
+import LiveCompositor from './compositor';
+
+export { WasmInstance, LiveCompositor };

ts/@live-compositor/web/src/input.ts

@@ -0,0 +1,5 @@
+import MP4Input from './mp4/input';
+
+type Input = { type: 'mp4' } & MP4Input;
+
+export default Input;