From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mail-yb1-xb2e.google.com (mail-yb1-xb2e.google.com [IPv6:2607:f8b0:4864:20::b2e]) by mail.toke.dk (Postfix) with ESMTPS id DF036A67F30 for ; Sun, 5 May 2024 20:56:54 +0200 (CEST) Authentication-Results: mail.toke.dk; dkim=pass (2048-bit key; unprotected) header.d=gmail.com header.i=@gmail.com header.a=rsa-sha256 header.s=20230601 header.b=h73+zHax Received: by mail-yb1-xb2e.google.com with SMTP id 3f1490d57ef6-de54b28c41eso1507359276.0 for ; Sun, 05 May 2024 11:56:54 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1714935409; x=1715540209; darn=lists.galene.org; h=to:subject:message-id:date:from:mime-version:from:to:cc:subject :date:message-id:reply-to; bh=A+7wQggePmwhxXkWZbzu/yY2gTxsE5pitvgJw39GEvI=; b=h73+zHaxOPOgvGWpxldgcgjQt8nfCdUqKQbMT6fue74nSIviiD1ETSIDN0XntEkPEw 7trzWU0phU5D/6XQMG/gHVGmvykYr0kzoJ8Lwq6va2v1PAGW4IPLnwmFJaAsXbsZhq6B AjbofTc1RU9vyTa+zH4sovIBaTwfCTdb/F0e6q0BlTwTfx6Lq9gKoxq3XXQu4OVlKMMt u36p5AEAkWoGCSXvcpZbOAGITYD6whkjfgNImC8lzi8+Q50OxvyKAPvlGdgySos6SeTw zexS3rCcPcV7C7wbyabooCWpVenx8nRhJYCdlvR7YbkpOf22L75lw2tenGh2iItSHDBn 3h9Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1714935409; x=1715540209; h=to:subject:message-id:date:from:mime-version:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=A+7wQggePmwhxXkWZbzu/yY2gTxsE5pitvgJw39GEvI=; b=v3mJjg6ujDDik1v1PGiYyQLXUlT0vjhcTGnrmIqGXEj7N2QB7toN6mh5tSlG8A2/a0 n30I1BE5CYz0XmYwGqqDkdzvXpg9qOzWqmBOMbuiL9XMI2+QSnBjxwxZ7MbwZXKdHcuc RnPq4apQzQ1LpNgwWHwGJR7Zfo8ctObgl0CDVG7KwzPpmWwbUTDxpUpnclD74wXAolsC Qb+wOSLblGKnqLQ8DvXPJcIHel3hfBtUrgnoO9nWEv5J5lNgOeKaBeRdWi0S5QzhuD4w s8z29y4+P79Ze3YWux7eZKYzzGoTuQn1LBsDtwzNobt0KBXQTC1KXIsOOaTJ72Z1+jCN Hrkg== X-Gm-Message-State: AOJu0YyLX5TTQTuikHmhms0BHni1sfgPEfI5lfzVDCkAJBHvPtGJT/LV D79oph2JJDfgge7GMkMg7KhX23Es1frXJpECj8Q2IRZwwwpaxaHCPGCZI8QHQnGK4V4vO8BaLfb Vh6x9MN/QqBKhjiG66LE7FFcp1P4l8584 X-Google-Smtp-Source: AGHT+IHkJXXxIEajEYbZ4df0YjTw4C40mjEnHxZEQ5myp1RaDL7E9/bOci838pUuwQnQu/D8yrhU1SpCWNyJo4C7Sz0= X-Received: by 2002:a05:6902:1b8f:b0:de6:1056:c9e7 with SMTP id ei15-20020a0569021b8f00b00de61056c9e7mr9552063ybb.8.1714935408622; Sun, 05 May 2024 11:56:48 -0700 (PDT) MIME-Version: 1.0 From: Francis Bolduc Date: Sun, 5 May 2024 14:56:37 -0400 Message-ID: To: galene@lists.galene.org Content-Type: text/plain; charset="UTF-8" Message-ID-Hash: 3AKTVD3OVA2TVO2QMXTROOVTUU4Q2FFW X-Message-ID-Hash: 3AKTVD3OVA2TVO2QMXTROOVTUU4Q2FFW X-MailFrom: fbolduc@gmail.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; emergency; loop; banned-address; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.9 Precedence: list Subject: [Galene] Blur background List-Id: =?utf-8?q?Gal=C3=A8ne_videoconferencing_server_discussion_list?= Archived-At: List-Archive: List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Hi, I made a little experiment to add "Blur Background" as a filter for the webcam. It is probably inefficient and insecure, but it works more-or-less for my use-cases. Also, please note that I am no Javascript programmer and I had to puke several times while dealing with the async cancer and impossible conversion between ImageBitmap and ImageData. So take the following patch for what it's worth. Comments are welcome. diff --git a/galene/static/galene.html b/galene/static/galene.html index 15b97b56..7230ecd5 100644 --- a/galene/static/galene.html +++ b/galene/static/galene.html @@ -295,6 +295,12 @@ + + + + + + diff --git a/galene/static/galene.js b/galene/static/galene.js index 963fb43a..ef062b40 100644 --- a/galene/static/galene.js +++ b/galene/static/galene.js @@ -35,6 +35,9 @@ let token = null; /** @type {boolean} */ let connectingAgain = false; +let offscreen; +let segmenter; + /** * @typedef {Object} settings * @property {boolean} [localMute] @@ -1051,10 +1054,10 @@ function Filter(stream, definition) { this.video.play(); if(this.definition.init) this.definition.init.call(this, this.context); - this.timer = setInterval(() => this.draw(), 1000 / this.frameRate); + this.timer = SetIntervalAsync.setIntervalAsync(async () => await this.draw(), 1000 / this.frameRate); } -Filter.prototype.draw = function() { +Filter.prototype.draw = async function() { // check framerate every 30 frames if((this.count % 30) === 0) { let frameRate = 0; @@ -1066,8 +1069,8 @@ Filter.prototype.draw = function() { } }); if(frameRate && frameRate != this.frameRate) { - clearInterval(this.timer); - this.timer = setInterval(() => this.draw(), 1000 / this.frameRate); + SetIntervalAsync.clearIntervalAsync(this.timer); + this.timer = SetIntervalAsync.setIntervalAsync(async () => await this.draw(), 1000 / this.frameRate); } } @@ -1092,7 +1095,7 @@ Filter.prototype.stop = function() { if(!this.timer) return; this.captureStream.getTracks()[0].stop(); - clearInterval(this.timer); + SetIntervalAsync.clearIntervalAsync(this.timer); this.timer = null; if(this.definition.cleanup) this.definition.cleanup.call(this); @@ -1132,13 +1135,46 @@ function setFilter(c) { c.userdata.filter = filter; } +async function blurBackground(src, width, height, ctx) { + + if(width == 0 || height == 0) { + return true; + } + + const bitmap = await createImageBitmap(src, 0, 0, width, height); + const segmentation = await segmenter.segmentPeople(bitmap); + + offscreen.getContext("2d").drawImage(bitmap, 0, 0, width, height); + const image = offscreen.getContext("2d").getImageData(0, 0, width, height); + + const foregroundThreshold = 0.5; // The minimum probability to color a pixel as foreground rather than background. Defaults to 0.5. Should be a number between 0 and 1. + const backgroundBlurAmount = 5; // How many pixels in the background blend into each other. Defaults to 3. Should be an integer between 1 and 20. + const edgeBlurAmount = 3; // How many pixels to blur on the edge between the person and the background by. Defaults to 3. Should be an integer between 0 and 20. + const flipHorizontal = false; // If the output should be flipped horizontally. Defaults to false. + bodySegmentation.drawBokehEffect( + ctx.canvas, + image, + segmentation, + foregroundThreshold, + backgroundBlurAmount, + edgeBlurAmount, + flipHorizontal + ); + + return true; +} + /** * @type {Object.} */ let filters = { + 'blur-background': { + description: "Blur Background", + f: blurBackground, + }, 'mirror-h': { description: "Horizontal mirror", - f: function(src, width, height, ctx) { + f: async function(src, width, height, ctx) { if(!(ctx instanceof CanvasRenderingContext2D)) throw new Error('bad context type'); if(ctx.canvas.width !== width || ctx.canvas.height !== height) { @@ -1153,7 +1189,7 @@ let filters = { }, 'mirror-v': { description: "Vertical mirror", - f: function(src, width, height, ctx) { + f: async function(src, width, height, ctx) { if(!(ctx instanceof CanvasRenderingContext2D)) throw new Error('bad context type'); if(ctx.canvas.width !== width || ctx.canvas.height !== height) { @@ -3905,6 +3941,18 @@ async function serverConnect() { } async function start() { + + offscreen = new OffscreenCanvas(1920, 1080); + + segmenter = await bodySegmentation.createSegmenter( + bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation, + { + runtime: 'mediapipe', + solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation', + modelType: 'general' + } + ); + try { let r = await fetch(".status") if(!r.ok) diff --git a/galene/webserver/webserver.go b/galene/webserver/webserver.go index 6fe611e3..b7f92fc5 100644 --- a/galene/webserver/webserver.go +++ b/galene/webserver/webserver.go @@ -98,18 +98,22 @@ func Serve(address string, dataDir string) error { } func cspHeader(w http.ResponseWriter, connect string) { - c := "connect-src ws: wss: 'self';" + c := "connect-src ws: wss: 'self' https://cdn.jsdelivr.net;" if connect != "" { - c = "connect-src " + connect + " ws: wss: 'self';" + c = "connect-src " + connect + " ws: wss: 'self' https://cdn.jsdelivr.net;" } w.Header().Add("Content-Security-Policy", - c+" img-src data: 'self'; media-src blob: 'self'; default-src 'self'") + c+" default-src 'self' 'wasm-unsafe-eval' https://cdn.jsdelivr.net") // Make browser stop sending referrer information w.Header().Add("Referrer-Policy", "no-referrer") // Require correct MIME type to load CSS and JS w.Header().Add("X-Content-Type-Options", "nosniff") + + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") } func notFound(w http.ResponseWriter) {