[Galene] Blur background

[Galene] Blur background

[Francis Bolduc]

Hi,

I made a little experiment to add "Blur Background" as a filter for
the webcam. It is probably inefficient and insecure, but it works
more-or-less for my use-cases. Also, please note that I am no
Javascript programmer and I had to puke several times while dealing
with the async cancer and impossible conversion between ImageBitmap
and ImageData. So take the following patch for what it's worth.

Comments are welcome.




diff --git a/galene/static/galene.html b/galene/static/galene.html
index 15b97b56..7230ecd5 100644
--- a/galene/static/galene.html
+++ b/galene/static/galene.html
@@ -295,6 +295,12 @@
         <button value="invite" value="invite">Invite</button>
     </dialog>

+    <script src="https://cdn.jsdelivr.net/npm/set-interval-async"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"
defer></script>
+    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl"
defer></script>
+    <script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/body-segmentation/dist/body-segmentation.js"
defer></script>
+    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation"
defer></script>
+
     <script src="/table/protocol.js" defer></script>
     <script src="/table/external/toastify/toastify.js" defer></script>
     <script src="/table/external/contextual/contextual.js" defer></script>
diff --git a/galene/static/galene.js b/galene/static/galene.js
index 963fb43a..ef062b40 100644
--- a/galene/static/galene.js
+++ b/galene/static/galene.js
@@ -35,6 +35,9 @@ let token = null;
 /** @type {boolean} */
 let connectingAgain = false;

+let offscreen;
+let segmenter;
+
 /**
  * @typedef {Object} settings
  * @property {boolean} [localMute]
@@ -1051,10 +1054,10 @@ function Filter(stream, definition) {
     this.video.play();
     if(this.definition.init)
         this.definition.init.call(this, this.context);
-    this.timer = setInterval(() => this.draw(), 1000 / this.frameRate);
+    this.timer = SetIntervalAsync.setIntervalAsync(async () => await
this.draw(), 1000 / this.frameRate);
 }

-Filter.prototype.draw = function() {
+Filter.prototype.draw = async function() {
     // check framerate every 30 frames
     if((this.count % 30) === 0) {
         let frameRate = 0;
@@ -1066,8 +1069,8 @@ Filter.prototype.draw = function() {
             }
         });
         if(frameRate && frameRate != this.frameRate) {
-            clearInterval(this.timer);
-            this.timer = setInterval(() => this.draw(), 1000 / this.frameRate);
+            SetIntervalAsync.clearIntervalAsync(this.timer);
+            this.timer = SetIntervalAsync.setIntervalAsync(async ()
=> await this.draw(), 1000 / this.frameRate);
         }
     }

@@ -1092,7 +1095,7 @@ Filter.prototype.stop = function() {
     if(!this.timer)
         return;
     this.captureStream.getTracks()[0].stop();
-    clearInterval(this.timer);
+    SetIntervalAsync.clearIntervalAsync(this.timer);
     this.timer = null;
     if(this.definition.cleanup)
         this.definition.cleanup.call(this);
@@ -1132,13 +1135,46 @@ function setFilter(c) {
     c.userdata.filter = filter;
 }

+async function blurBackground(src, width, height, ctx) {
+
+    if(width == 0 || height == 0) {
+        return true;
+    }
+
+    const bitmap = await createImageBitmap(src, 0, 0, width, height);
+    const segmentation = await segmenter.segmentPeople(bitmap);
+
+    offscreen.getContext("2d").drawImage(bitmap, 0, 0, width, height);
+    const image = offscreen.getContext("2d").getImageData(0, 0, width, height);
+
+    const foregroundThreshold = 0.5;  // The minimum probability to
color a pixel as foreground rather than background. Defaults to 0.5.
Should be a number between 0 and 1.
+    const backgroundBlurAmount = 5;   // How many pixels in the
background blend into each other. Defaults to 3. Should be an integer
between 1 and 20.
+    const edgeBlurAmount = 3;         // How many pixels to blur on
the edge between the person and the background by. Defaults to 3.
Should be an integer between 0 and 20.
+    const flipHorizontal = false;     // If the output should be
flipped horizontally. Defaults to false.
+    bodySegmentation.drawBokehEffect(
+        ctx.canvas,
+        image,
+        segmentation,
+        foregroundThreshold,
+        backgroundBlurAmount,
+        edgeBlurAmount,
+        flipHorizontal
+    );
+
+    return true;
+}
+
 /**
  * @type {Object.<string,filterDefinition>}
  */
 let filters = {
+    'blur-background': {
+        description: "Blur Background",
+        f: blurBackground,
+    },
     'mirror-h': {
         description: "Horizontal mirror",
-        f: function(src, width, height, ctx) {
+        f: async function(src, width, height, ctx) {
             if(!(ctx instanceof CanvasRenderingContext2D))
                 throw new Error('bad context type');
             if(ctx.canvas.width !== width || ctx.canvas.height !== height) {
@@ -1153,7 +1189,7 @@ let filters = {
     },
     'mirror-v': {
         description: "Vertical mirror",
-        f: function(src, width, height, ctx) {
+        f: async function(src, width, height, ctx) {
             if(!(ctx instanceof CanvasRenderingContext2D))
                 throw new Error('bad context type');
             if(ctx.canvas.width !== width || ctx.canvas.height !== height) {
@@ -3905,6 +3941,18 @@ async function serverConnect() {
 }

 async function start() {
+
+    offscreen = new OffscreenCanvas(1920, 1080);
+
+    segmenter = await bodySegmentation.createSegmenter(
+        bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation,
+        {
+            runtime: 'mediapipe',
+            solutionPath:
'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation',
+            modelType: 'general'
+        }
+    );
+
     try {
         let r = await fetch(".status")
         if(!r.ok)
diff --git a/galene/webserver/webserver.go b/galene/webserver/webserver.go
index 6fe611e3..b7f92fc5 100644
--- a/galene/webserver/webserver.go
+++ b/galene/webserver/webserver.go
@@ -98,18 +98,22 @@ func Serve(address string, dataDir string) error {
 }

 func cspHeader(w http.ResponseWriter, connect string) {
-    c := "connect-src ws: wss: 'self';"
+    c := "connect-src ws: wss: 'self' https://cdn.jsdelivr.net;"
     if connect != "" {
-        c = "connect-src " + connect + " ws: wss: 'self';"
+        c = "connect-src " + connect + " ws: wss: 'self'
https://cdn.jsdelivr.net;"
     }
     w.Header().Add("Content-Security-Policy",
-        c+" img-src data: 'self'; media-src blob: 'self'; default-src 'self'")
+        c+" default-src 'self' 'wasm-unsafe-eval' https://cdn.jsdelivr.net")

     // Make browser stop sending referrer information
     w.Header().Add("Referrer-Policy", "no-referrer")

     // Require correct MIME type to load CSS and JS
     w.Header().Add("X-Content-Type-Options", "nosniff")
+
+    w.Header().Set("Access-Control-Allow-Origin", "*")
+    w.Header().Set("Access-Control-Allow-Methods", "GET")
+    w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
 }

 func notFound(w http.ResponseWriter) {

[Galene] Re: Blur background

[Juliusz Chroboczek]

Hello Francis,

I'd really like to have background blur in Galene, but I don't see any
good way to do it without impairing the user's privacy.

> +    <script src="https://cdn.jsdelivr.net/npm/set-interval-async"></script>
> +    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"
> defer></script>
> +    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl"
> defer></script>
> +    <script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/body-segmentation/dist/body-segmentation.js"
> defer></script>
> +    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation"
> defer></script>

That's problematic.  One of the design criteria of Galene is that it
preserves the user's privacy.  These links imply that everytime somebody
connects to any instance of Galene in the workld, the jsdelivr.net
distribution network is informed of the fact.

So either we bundle all the scripts in Galene, or this cannot go in.

>  async function start() {
> +
> +    offscreen = new OffscreenCanvas(1920, 1080);
> +
> +    segmenter = await bodySegmentation.createSegmenter(
> +        bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation,
> +        {
> +            runtime: 'mediapipe',
> +            solutionPath:
> 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation',
> +            modelType: 'general'
> +        }
> +    );

This should be done lazily, in the filter, not at startup.

> +    w.Header().Set("Access-Control-Allow-Origin", "*")
> +    w.Header().Set("Access-Control-Allow-Methods", "GET")
> +    w.Header().Set("Access-Control-Allow-Headers", "Content-Type")

That's a potential security hole, since it makes Galene vulnerable to
cross-origin scripting attacks.  What we do in Galene is to only set these
headers conditionally, in places that are known to be safe.  See for example

  https://github.com/jech/galene/blob/master/webserver/whip.go#L178

-- Juliusz

[Galene] Re: Blur background

[Francis Bolduc]

Juliusz,

Yes, loading 3rdparty libraries is conceptually insecure. I initially
tried to download the Javascript libraries and wasm modules to serve
them via Galene, but failed because they refer to each other via full
URL instead of relative paths when you load them. I also tried to
download their source code and build them myself, but my head exploded
when I saw the kind of build system those libraries have. So I
resorted to this hack for my own purposes, knowing full well that it
would not be merged.

> >  async function start() {
> > +
> > +    offscreen = new OffscreenCanvas(1920, 1080);
> > +
> > +    segmenter = await bodySegmentation.createSegmenter(
> > +        bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation,
> > +        {
> > +            runtime: 'mediapipe',
> > +            solutionPath:
> > 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation',
> > +            modelType: 'general'
> > +        }
> > +    );
>
> This should be done lazily, in the filter, not at startup.

Not sure what you mean. How would you do this?

[Galene] Re: Blur background

[Juliusz Chroboczek]

> Yes, loading 3rdparty libraries is conceptually insecure. I initially
> tried to download the Javascript libraries and wasm modules to serve
> them via Galene, but failed because they refer to each other via full
> URL instead of relative paths when you load them.  I also tried to
> download their source code and build them myself, but my head exploded
> when I saw the kind of build system those libraries have.

Yeah, these are the two reasons I've been waiting for Chrome to implement
native background blur [1], but that effort appears to have been abandoned.

[1] https://chromestatus.com/feature/5077577782263808

>>> +    offscreen = new OffscreenCanvas(1920, 1080);

>> This should be done lazily, in the filter, not at startup.

> Not sure what you mean. How would you do this?

Two ways.  One is to add a function 'init' to your filter definition:

    'blur-background':
        description: "Blur Background",
        init: initBlurBackground,
        f: blurBackground,
    },

and then, in initBlurBackground, instantiate a new field in your filter:

    function initBlurBackground(ctx) {
        this.offscreen = new OffscreenCanvas(...)
        ...

The other solution is to simply do the initialisation in the main function
of the filter:

    if(!(offscreen in this)) {
        this.offscreen = new OffscreenCanvas(...)
        ...

In either case, the data structures are only created when the filter is
first invoked, which means that only users who use your filter pay the
cost of initialisation.

-- Juliusz

[Galene] Re: Blur background

[Francis Bolduc]

Thank you, I did not notice the init method. I will use that one.

Regards,
Francis


On Sun, May 5, 2024 at 4:47 PM Juliusz Chroboczek <jch@irif.fr> wrote:
>
> > Yes, loading 3rdparty libraries is conceptually insecure. I initially
> > tried to download the Javascript libraries and wasm modules to serve
> > them via Galene, but failed because they refer to each other via full
> > URL instead of relative paths when you load them.  I also tried to
> > download their source code and build them myself, but my head exploded
> > when I saw the kind of build system those libraries have.
>
> Yeah, these are the two reasons I've been waiting for Chrome to implement
> native background blur [1], but that effort appears to have been abandoned.
>
> [1] https://chromestatus.com/feature/5077577782263808
>
> >>> +    offscreen = new OffscreenCanvas(1920, 1080);
>
> >> This should be done lazily, in the filter, not at startup.
>
> > Not sure what you mean. How would you do this?
>
> Two ways.  One is to add a function 'init' to your filter definition:
>
>     'blur-background':
>         description: "Blur Background",
>         init: initBlurBackground,
>         f: blurBackground,
>     },
>
> and then, in initBlurBackground, instantiate a new field in your filter:
>
>     function initBlurBackground(ctx) {
>         this.offscreen = new OffscreenCanvas(...)
>         ...
>
> The other solution is to simply do the initialisation in the main function
> of the filter:
>
>     if(!(offscreen in this)) {
>         this.offscreen = new OffscreenCanvas(...)
>         ...
>
> In either case, the data structures are only created when the filter is
> first invoked, which means that only users who use your filter pay the
> cost of initialisation.
>
> -- Juliusz