[Galene] Blur background

[Francis Bolduc <fbolduc@gmail.com>]

Hi,

I made a little experiment to add "Blur Background" as a filter for
the webcam. It is probably inefficient and insecure, but it works
more-or-less for my use-cases. Also, please note that I am no
Javascript programmer and I had to puke several times while dealing
with the async cancer and impossible conversion between ImageBitmap
and ImageData. So take the following patch for what it's worth.

Comments are welcome.




diff --git a/galene/static/galene.html b/galene/static/galene.html
index 15b97b56..7230ecd5 100644
--- a/galene/static/galene.html
+++ b/galene/static/galene.html
@@ -295,6 +295,12 @@
         <button value="invite" value="invite">Invite</button>
     </dialog>

+    <script src="https://cdn.jsdelivr.net/npm/set-interval-async"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"
defer></script>
+    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl"
defer></script>
+    <script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/body-segmentation/dist/body-segmentation.js"
defer></script>
+    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation"
defer></script>
+
     <script src="/table/protocol.js" defer></script>
     <script src="/table/external/toastify/toastify.js" defer></script>
     <script src="/table/external/contextual/contextual.js" defer></script>
diff --git a/galene/static/galene.js b/galene/static/galene.js
index 963fb43a..ef062b40 100644
--- a/galene/static/galene.js
+++ b/galene/static/galene.js
@@ -35,6 +35,9 @@ let token = null;
 /** @type {boolean} */
 let connectingAgain = false;

+let offscreen;
+let segmenter;
+
 /**
  * @typedef {Object} settings
  * @property {boolean} [localMute]
@@ -1051,10 +1054,10 @@ function Filter(stream, definition) {
     this.video.play();
     if(this.definition.init)
         this.definition.init.call(this, this.context);
-    this.timer = setInterval(() => this.draw(), 1000 / this.frameRate);
+    this.timer = SetIntervalAsync.setIntervalAsync(async () => await
this.draw(), 1000 / this.frameRate);
 }

-Filter.prototype.draw = function() {
+Filter.prototype.draw = async function() {
     // check framerate every 30 frames
     if((this.count % 30) === 0) {
         let frameRate = 0;
@@ -1066,8 +1069,8 @@ Filter.prototype.draw = function() {
             }
         });
         if(frameRate && frameRate != this.frameRate) {
-            clearInterval(this.timer);
-            this.timer = setInterval(() => this.draw(), 1000 / this.frameRate);
+            SetIntervalAsync.clearIntervalAsync(this.timer);
+            this.timer = SetIntervalAsync.setIntervalAsync(async ()
=> await this.draw(), 1000 / this.frameRate);
         }
     }

@@ -1092,7 +1095,7 @@ Filter.prototype.stop = function() {
     if(!this.timer)
         return;
     this.captureStream.getTracks()[0].stop();
-    clearInterval(this.timer);
+    SetIntervalAsync.clearIntervalAsync(this.timer);
     this.timer = null;
     if(this.definition.cleanup)
         this.definition.cleanup.call(this);
@@ -1132,13 +1135,46 @@ function setFilter(c) {
     c.userdata.filter = filter;
 }

+async function blurBackground(src, width, height, ctx) {
+
+    if(width == 0 || height == 0) {
+        return true;
+    }
+
+    const bitmap = await createImageBitmap(src, 0, 0, width, height);
+    const segmentation = await segmenter.segmentPeople(bitmap);
+
+    offscreen.getContext("2d").drawImage(bitmap, 0, 0, width, height);
+    const image = offscreen.getContext("2d").getImageData(0, 0, width, height);
+
+    const foregroundThreshold = 0.5;  // The minimum probability to
color a pixel as foreground rather than background. Defaults to 0.5.
Should be a number between 0 and 1.
+    const backgroundBlurAmount = 5;   // How many pixels in the
background blend into each other. Defaults to 3. Should be an integer
between 1 and 20.
+    const edgeBlurAmount = 3;         // How many pixels to blur on
the edge between the person and the background by. Defaults to 3.
Should be an integer between 0 and 20.
+    const flipHorizontal = false;     // If the output should be
flipped horizontally. Defaults to false.
+    bodySegmentation.drawBokehEffect(
+        ctx.canvas,
+        image,
+        segmentation,
+        foregroundThreshold,
+        backgroundBlurAmount,
+        edgeBlurAmount,
+        flipHorizontal
+    );
+
+    return true;
+}
+
 /**
  * @type {Object.<string,filterDefinition>}
  */
 let filters = {
+    'blur-background': {
+        description: "Blur Background",
+        f: blurBackground,
+    },
     'mirror-h': {
         description: "Horizontal mirror",
-        f: function(src, width, height, ctx) {
+        f: async function(src, width, height, ctx) {
             if(!(ctx instanceof CanvasRenderingContext2D))
                 throw new Error('bad context type');
             if(ctx.canvas.width !== width || ctx.canvas.height !== height) {
@@ -1153,7 +1189,7 @@ let filters = {
     },
     'mirror-v': {
         description: "Vertical mirror",
-        f: function(src, width, height, ctx) {
+        f: async function(src, width, height, ctx) {
             if(!(ctx instanceof CanvasRenderingContext2D))
                 throw new Error('bad context type');
             if(ctx.canvas.width !== width || ctx.canvas.height !== height) {
@@ -3905,6 +3941,18 @@ async function serverConnect() {
 }

 async function start() {
+
+    offscreen = new OffscreenCanvas(1920, 1080);
+
+    segmenter = await bodySegmentation.createSegmenter(
+        bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation,
+        {
+            runtime: 'mediapipe',
+            solutionPath:
'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation',
+            modelType: 'general'
+        }
+    );
+
     try {
         let r = await fetch(".status")
         if(!r.ok)
diff --git a/galene/webserver/webserver.go b/galene/webserver/webserver.go
index 6fe611e3..b7f92fc5 100644
--- a/galene/webserver/webserver.go
+++ b/galene/webserver/webserver.go
@@ -98,18 +98,22 @@ func Serve(address string, dataDir string) error {
 }

 func cspHeader(w http.ResponseWriter, connect string) {
-    c := "connect-src ws: wss: 'self';"
+    c := "connect-src ws: wss: 'self' https://cdn.jsdelivr.net;"
     if connect != "" {
-        c = "connect-src " + connect + " ws: wss: 'self';"
+        c = "connect-src " + connect + " ws: wss: 'self'
https://cdn.jsdelivr.net;"
     }
     w.Header().Add("Content-Security-Policy",
-        c+" img-src data: 'self'; media-src blob: 'self'; default-src 'self'")
+        c+" default-src 'self' 'wasm-unsafe-eval' https://cdn.jsdelivr.net")

     // Make browser stop sending referrer information
     w.Header().Add("Referrer-Policy", "no-referrer")

     // Require correct MIME type to load CSS and JS
     w.Header().Add("X-Content-Type-Options", "nosniff")
+
+    w.Header().Set("Access-Control-Allow-Origin", "*")
+    w.Header().Set("Access-Control-Allow-Methods", "GET")
+    w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
 }

 func notFound(w http.ResponseWriter) {