|
--- |
|
library_name: transformers.js |
|
tags: |
|
- pose-estimation |
|
license: apache-2.0 |
|
--- |
|
|
|
|
|
https://github.com/open-mmlab/mmpose/tree/main/projects/rtmo with ONNX weights to be compatible with Transformers.js. |
|
|
|
## Usage (Transformers.js) |
|
|
|
If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@xenova/transformers) using: |
|
```bash |
|
npm i @xenova/transformers |
|
``` |
|
|
|
**Example:** Perform pose-estimation w/ `Xenova/RTMO-l`. |
|
|
|
```js |
|
import { AutoModel, AutoProcessor, RawImage } from '@xenova/transformers'; |
|
|
|
// Load model and processor |
|
const model_id = 'Xenova/RTMO-l'; |
|
const model = await AutoModel.from_pretrained(model_id); |
|
const processor = await AutoProcessor.from_pretrained(model_id); |
|
|
|
// Read image and run processor |
|
const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'; |
|
const image = await RawImage.read(url); |
|
const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); |
|
|
|
// Predict bounding boxes and keypoints |
|
const { dets, keypoints } = await model({ input: pixel_values }); |
|
|
|
// Select the first image |
|
const predicted_boxes = dets.tolist()[0]; |
|
const predicted_points = keypoints.tolist()[0]; |
|
const [height, width] = original_sizes[0]; |
|
const [resized_height, resized_width] = reshaped_input_sizes[0]; |
|
|
|
// Compute scale values |
|
const xScale = width / resized_width; |
|
const yScale = height / resized_height; |
|
|
|
// Define thresholds |
|
const point_threshold = 0.3; |
|
const box_threshold = 0.3; |
|
|
|
// Display results |
|
for (let i = 0; i < predicted_boxes.length; ++i) { |
|
const [xmin, ymin, xmax, ymax, box_score] = predicted_boxes[i]; |
|
if (box_score < box_threshold) continue; |
|
|
|
const x1 = (xmin * xScale).toFixed(2); |
|
const y1 = (ymin * yScale).toFixed(2); |
|
const x2 = (xmax * xScale).toFixed(2); |
|
const y2 = (ymax * yScale).toFixed(2); |
|
|
|
console.log(`Found person at [${x1}, ${y1}, ${x2}, ${y2}] with score ${box_score.toFixed(3)}`) |
|
const points = predicted_points[i]; // of shape [17, 3] |
|
for (let id = 0; id < points.length; ++id) { |
|
const label = model.config.id2label[id]; |
|
const [x, y, point_score] = points[id]; |
|
if (point_score < point_threshold) continue; |
|
console.log(` - ${label}: (${(x * xScale).toFixed(2)}, ${(y * yScale).toFixed(2)}) with score ${point_score.toFixed(3)}`); |
|
} |
|
} |
|
``` |
|
|
|
<details> |
|
|
|
<summary>See example output</summary> |
|
|
|
``` |
|
Found person at [400.13, 66.05, 657.48, 496.67] with score 0.978 |
|
- nose: (520.40, 118.17) with score 0.445 |
|
- left_eye: (531.83, 111.10) with score 0.350 |
|
- left_shoulder: (559.65, 168.66) with score 0.999 |
|
- right_shoulder: (469.70, 160.04) with score 0.999 |
|
- left_elbow: (573.20, 237.82) with score 1.000 |
|
- right_elbow: (438.51, 218.06) with score 0.999 |
|
- left_wrist: (604.74, 308.75) with score 0.999 |
|
- right_wrist: (495.52, 219.24) with score 0.999 |
|
- left_hip: (537.36, 306.24) with score 1.000 |
|
- right_hip: (477.61, 314.79) with score 0.998 |
|
- left_knee: (576.44, 360.67) with score 1.000 |
|
- right_knee: (500.26, 448.33) with score 0.997 |
|
- left_ankle: (575.94, 461.43) with score 0.998 |
|
- right_ankle: (525.18, 436.10) with score 0.996 |
|
Found person at [84.74, 11.57, 524.53, 535.62] with score 0.970 |
|
- left_shoulder: (240.00, 106.15) with score 0.998 |
|
- right_shoulder: (230.72, 131.27) with score 0.999 |
|
- left_elbow: (319.58, 164.42) with score 0.999 |
|
- right_elbow: (232.16, 226.10) with score 1.000 |
|
- left_wrist: (390.95, 220.65) with score 0.999 |
|
- right_wrist: (157.61, 227.93) with score 0.999 |
|
- left_hip: (363.29, 249.14) with score 1.000 |
|
- right_hip: (337.65, 250.50) with score 1.000 |
|
- left_knee: (297.35, 368.55) with score 1.000 |
|
- right_knee: (328.29, 390.84) with score 1.000 |
|
- left_ankle: (433.81, 343.83) with score 0.999 |
|
- right_ankle: (452.74, 504.60) with score 0.995 |
|
Found person at [-4.11, 53.42, 174.91, 372.64] with score 0.644 |
|
- nose: (74.67, 84.38) with score 0.375 |
|
- left_shoulder: (114.29, 113.60) with score 0.991 |
|
- right_shoulder: (44.21, 117.73) with score 0.989 |
|
- left_elbow: (124.69, 159.42) with score 0.978 |
|
- right_elbow: (26.54, 154.78) with score 0.995 |
|
- left_wrist: (132.86, 168.78) with score 0.957 |
|
- right_wrist: (6.44, 195.67) with score 0.986 |
|
- left_hip: (98.90, 199.49) with score 0.978 |
|
- right_hip: (62.77, 200.49) with score 0.976 |
|
- left_knee: (111.91, 277.06) with score 0.998 |
|
- right_knee: (65.08, 276.40) with score 0.999 |
|
- left_ankle: (128.95, 344.65) with score 0.973 |
|
- right_ankle: (63.55, 345.60) with score 0.992 |
|
Found person at [511.40, 32.53, 658.71, 345.63] with score 0.384 |
|
- nose: (554.88, 74.25) with score 0.796 |
|
- left_eye: (563.64, 68.39) with score 0.716 |
|
- right_eye: (547.38, 68.22) with score 0.542 |
|
- left_ear: (575.42, 72.40) with score 0.324 |
|
- left_shoulder: (576.47, 105.27) with score 0.999 |
|
- right_shoulder: (531.19, 105.55) with score 0.956 |
|
- left_elbow: (623.35, 151.54) with score 0.999 |
|
- right_elbow: (549.79, 144.36) with score 0.387 |
|
- left_wrist: (631.33, 198.37) with score 0.991 |
|
- right_wrist: (547.36, 162.58) with score 0.486 |
|
- left_hip: (578.36, 192.67) with score 0.989 |
|
- right_hip: (555.21, 188.00) with score 0.925 |
|
- left_knee: (604.56, 239.95) with score 0.977 |
|
- right_knee: (545.23, 221.37) with score 0.952 |
|
- left_ankle: (587.82, 323.26) with score 0.401 |
|
- right_ankle: (546.77, 322.69) with score 0.846 |
|
``` |
|
|
|
</details> |
