<!DOCTYPE html> <html> <head> <meta charset="UTF-8"> <title>Program that uses Handtrackjs</title> <script src="https://unpkg.com/carbon-components@latest/scripts/carbon-components.js"></script> <script src="https://cdn.jsdelivr.net/npm/handtrackjs/dist/handtrack.min.js"> </script> <script> /* * Based on https://codepen.io/victordibia/pen/RdWbEY * Also see: * https://victordibia.com/handtrack.js/ * https://devpost.com/software/handtrack-js-1-0-real-time-handtracking-in-the-browser# */ /* ********************************************************************** * Main = General purpose skeleton program for using handtrack in any application ********************************************************************** */ class Main { //*1 Main class = general purpose skeleton program constructor () { //*1 // Constants and state variables this.isVideo = false; this.model = null; this.modelParams = { //*1 flipHorizontal: true, // flip e.g for video maxNumBoxes: 20, // maximum number of boxes to detect iouThreshold: 0.5, // ioU threshold for non-max suppression scoreThreshold: 0.6, // confidence threshold for predictions. } this.demo = new Demo (); //*2 Instantiate your application program // Must put this in a separate method cause constructors can't be async this.init() //*3 Separate method cause constructors can't be async } async init () { //*3 // Load the model (and wait here till it's loaded) this.model = await handTrack.load (this.modelParams) //*3 updatenote.innerText = "Loaded Model!" // Start running let status = await handTrack.startVideo (myvideo) //*3 if (status) { updatenote.innerText = "Video started. Now tracking" this.isVideo = true; this.runDetection() ; //*4 Then start running on each frame } else { updatenote.innerText = "Please enable video"; } } async runDetection() { //*4 let predictions = await this.model.detect (myvideo) //*4 // Our stuff this.demo.doit (predictions) //*4 // Display the values in the HTML this.showPredictions (predictions) //*5 Also display the prediction values in the HTML // Also display the predictions graphically this.model.renderPredictions(predictions, canvas, canvas.getContext("2d"), myvideo); // Set up for next round; NB must use "=>" so can access "this" if (this.isVideo) requestAnimationFrame(() => this.runDetection()); //*4 } // Display the prediction values in the HTML showPredictions (predictions) { //*5 results.innerText = "" for (let p of predictions) { //*5 results.innerText += p.label + ": " + p.bbox.map(Math.round) + "\n" //*5 } } } /* ********************************************************************** * Demo = A demo application that responds to hand position in a simple way * Closed hand drags the ball, open hand is ignored * * You can substitute your own application here. * Just accept a call to doit(predictions) * which will contain an array of all the current predictions, * and do whatever you want with them. * * "predictions" is an array, each element contains a dict like { * bbox: [x, y, w, h], * class: (is redundant with label) * label: "open" or "closed" or "pinch" or "point" or "face" or "tip" or "pinchtip" * score: 0..1 confidence * } ********************************************************************** */ class Demo { //*6 Demo application program constructor () { //*6 // Take moving average of last n, to smooth the input this.n = 8 //*7 Use moving average to smooth this.lastn = [] //*7 } doit (predictions) { //*8 Main calls us when there is a new prediction for (let p of predictions) { //*9 We respond only to closed hand if (p.label == "closed") { //*9 // Update the moving average array if (this.lastn.length > this.n) this.lastn.shift(); //*7 this.lastn.push (p.bbox); //*7 // Cute (functional) way to compute means const x = this.lastn.map ((x) => x[0]) .reduce ((x,y) => x+y) / this.lastn.length //*10 Functional programming formula for mean const y = this.lastn.map ((x) => x[1]) .reduce ((x,y) => x+y) / this.lastn.length const w = this.lastn.map ((x) => x[2]) .reduce ((x,y) => x+y) / this.lastn.length const h = this.lastn.map ((x) => x[3]) .reduce ((x,y) => x+y) / this.lastn.length this.moveBall (x+w/2, y+h/2, (w+h)/2) //*11 Move the ball // Just go with the first closed hand we find, then quit break } } } // Move the ball to x, y with size z moveBall (x, y, z) { //*11 const gc = demo.getContext("2d") // Background gc.fillStyle = "lightblue" gc.beginPath() gc.rect(0, 0, demo.width, demo.height) gc.fill() // Ball gc.fillStyle = "red" //*11 gc.beginPath() //*11 gc.arc(x, y, 0.5*z, 0, 2*Math.PI); //*11 gc.fill() //*11 } } window.onload = function () { new Main () } </script> </head> <body> <video autoplay="autoplay" id="myvideo"></video> <canvas id="canvas"></canvas> <div id="updatenote"> loading model ..</div> <br/> <canvas id="demo" width="640" height="480"></canvas> <br/> <div id="results"></div> </body> </html>