Handtrackjs: Usage example code: usehandtrackjs.html

<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Program that uses Handtrackjs</title>

<script src="https://unpkg.com/carbon-components@latest/scripts/carbon-components.js"></script>
<script src="https://cdn.jsdelivr.net/npm/handtrackjs/dist/handtrack.min.js"> </script>

<script>
/*
* Based on https://codepen.io/victordibia/pen/RdWbEY 
* Also see:
* https://victordibia.com/handtrack.js/
* https://devpost.com/software/handtrack-js-1-0-real-time-handtracking-in-the-browser#
*/

/*
 **********************************************************************
 * Main = General purpose skeleton program for using handtrack in any application
 **********************************************************************
 */
class Main { //*1 Main class = general purpose skeleton program
    constructor () { //*1
	// Constants and state variables
	this.isVideo = false;
	this.model = null;
	this.modelParams = { //*1
	    flipHorizontal: true,   // flip e.g for video  
	    maxNumBoxes: 20,        // maximum number of boxes to detect
	    iouThreshold: 0.5,      // ioU threshold for non-max suppression
	    scoreThreshold: 0.6,    // confidence threshold for predictions.
	}

	this.demo = new Demo (); //*2 Instantiate your application program

	// Must put this in a separate method cause constructors can't be async
	this.init() //*3 Separate method cause constructors can't be async
    }

    async init () { //*3
	// Load the model (and wait here till it's loaded)
	this.model = await handTrack.load (this.modelParams) //*3
	updatenote.innerText = "Loaded Model!"

	// Start running
	let status = await handTrack.startVideo (myvideo) //*3
	if (status) {
	    updatenote.innerText = "Video started. Now tracking"
	    this.isVideo = true;
	    this.runDetection() ; //*4 Then start running on each frame
	} else {
	    updatenote.innerText = "Please enable video";
	}
    }

    async runDetection() { //*4
	let predictions = await this.model.detect (myvideo) //*4

	// Our stuff
	this.demo.doit (predictions) //*4

	// Display the values in the HTML
	this.showPredictions (predictions) //*5 Also display the prediction values in the HTML

	// Also display the predictions graphically
	this.model.renderPredictions(predictions, canvas, canvas.getContext("2d"), myvideo);

	// Set up for next round; NB must use "=>" so can access "this"
	if (this.isVideo) requestAnimationFrame(() => this.runDetection()); //*4
    }

    // Display the prediction values in the HTML
    showPredictions (predictions) { //*5
	results.innerText = ""
	for (let p of predictions) { //*5
	    results.innerText += p.label + ":  " + p.bbox.map(Math.round) + "\n" //*5
	}
    }
}

/*
 **********************************************************************
 * Demo = A demo application that responds to hand position in a simple way
 * Closed hand drags the ball, open hand is ignored
 *
 * You can substitute your own application here.
 * Just accept a call to doit(predictions) 
 * which will contain an array of all the current predictions,
 * and do whatever you want with them.
 *
 * "predictions" is an array, each element contains a dict like {
 *	bbox: [x, y, w, h],
 *	class: (is redundant with label)
 *	label: "open" or "closed" or "pinch" or "point" or "face" or "tip" or "pinchtip"
 *	score: 0..1 confidence
 * }
 **********************************************************************
 */
class Demo { //*6 Demo application program
    constructor () { //*6
	// Take moving average of last n, to smooth the input
	this.n = 8 //*7 Use moving average to smooth
	this.lastn = [] //*7
    }
    
    doit (predictions) { //*8 Main calls us when there is a new prediction
	for (let p of predictions) { //*9 We respond only to closed hand
	    if (p.label == "closed") { //*9
		// Update the moving average array
		if (this.lastn.length > this.n) this.lastn.shift(); //*7
		this.lastn.push (p.bbox); //*7
		
		// Cute (functional) way to compute means
		const x = this.lastn.map ((x) => x[0]) .reduce ((x,y) => x+y) / this.lastn.length //*10 Functional programming formula for mean
		const y = this.lastn.map ((x) => x[1]) .reduce ((x,y) => x+y) / this.lastn.length
		const w = this.lastn.map ((x) => x[2]) .reduce ((x,y) => x+y) / this.lastn.length
		const h = this.lastn.map ((x) => x[3]) .reduce ((x,y) => x+y) / this.lastn.length

		this.moveBall (x+w/2, y+h/2, (w+h)/2) //*11 Move the ball

		// Just go with the first closed hand we find, then quit
		break
	    }
	}
    }

    // Move the ball to x, y with size z
    moveBall (x, y, z) { //*11
	const gc = demo.getContext("2d")

	// Background
	gc.fillStyle = "lightblue"
	gc.beginPath()
	gc.rect(0, 0, demo.width, demo.height)
	gc.fill()

	// Ball
	gc.fillStyle = "red" //*11
	gc.beginPath() //*11
	gc.arc(x, y, 0.5*z, 0, 2*Math.PI); //*11
	gc.fill() //*11
    }
}

window.onload = function () { new Main () }

</script>

</head>

<body>
    <video autoplay="autoplay" id="myvideo"></video>
    <canvas id="canvas"></canvas>
    <div id="updatenote"> loading model ..</div>
    <br/>
    <canvas id="demo" width="640" height="480"></canvas>
    <br/>
    <div id="results"></div>
</body>
</html>
[download file]