hg_sync.cpp File Reference

Mercury hand tracking main file. More...

#include "hg_sync.hpp"
#include "hg_image_math.inl"
#include "tracking/t_hand_tracking.h"
#include "util/u_box_iou.hpp"
#include "util/u_hand_tracking.h"
#include "math/m_vec2.h"
#include "util/u_misc.h"
#include "xrt/xrt_defines.h"
#include "xrt/xrt_frame.h"
#include "xrt/xrt_tracking.h"
#include <numeric>

Include dependency graph for hg_sync.cpp:

Functions
static bool	xrt::tracking::hand::mercury::getCalibration (struct HandTracking *hgt, t_stereo_camera_calibration &calibration)
	Setup helper functions.
static bool	xrt::tracking::hand::mercury::check_outside_view (struct HandTracking *hgt, struct t_camera_extra_info_one_view boundary, xrt_vec2 &keypoint)
static void	xrt::tracking::hand::mercury::back_project (struct HandTracking *hgt, Eigen::Array< float, 3, 21 > &pts, int hand_idx, bool also_debug_output, int num_outside[2])
static void	xrt::tracking::hand::mercury::back_project_keypoint_output (struct HandTracking *hgt, int hand_idx, int view_idx)
static bool	xrt::tracking::hand::mercury::handle_changed_image_size (HandTracking *hgt, xrt_size &new_one_view_size)
float	xrt::tracking::hand::mercury::hand_confidence_value (float reprojection_error, one_frame_input &input)
xrt_vec3	xrt::tracking::hand::mercury::correct_direction (xrt_vec2 in)
void	xrt::tracking::hand::mercury::check_new_user_event (struct HandTracking *hgt)
static float	xrt::tracking::hand::mercury::hand_bounding_boxes_iou (const hand_region_of_interest &one, const hand_region_of_interest &two)
void	xrt::tracking::hand::mercury::dispatch_and_process_hand_detections (struct HandTracking *hgt)
void	xrt::tracking::hand::mercury::hand_joint_set_to_eigen_21 (const xrt_hand_joint_set &set, Eigen::Array< float, 3, 21 > &out)
void	xrt::tracking::hand::mercury::predict_new_regions_of_interest (struct HandTracking *hgt)
void	xrt::tracking::hand::mercury::stop_everything_if_hands_are_overlapping (struct HandTracking *hgt)
bool	xrt::tracking::hand::mercury::hand_too_far (struct HandTracking *hgt, xrt_hand_joint_set &set)
void	xrt::tracking::hand::mercury::scribble_image_boundary (struct HandTracking *hgt)
t_hand_tracking_sync *	t_hand_tracking_sync_mercury_create (struct t_stereo_camera_calibration *calib, struct t_hand_tracking_create_info create_info, const char *models_folder)
	Create a Mercury hand tracking pipeline.

Variables
static enum xrt_space_relation_flags	xrt::tracking::hand::mercury::valid_flags_ht

Detailed Description

Mercury hand tracking main file.

Author

Jakob Bornecrantz jakob.nosp@m.@col.nosp@m.labor.nosp@m.a.co.nosp@m.m

Moshi Turner moshi.nosp@m.turn.nosp@m.er@pr.nosp@m.oton.nosp@m.mail..nosp@m.com

Charlton Rodda charl.nosp@m.ton..nosp@m.rodda.nosp@m.@col.nosp@m.labor.nosp@m.a.co.nosp@m.m

Function Documentation

back_project_keypoint_output()

static void xrt::tracking::hand::mercury::back_project_keypoint_output ( struct HandTracking *  hgt, int  hand_idx, int  view_idx  )

static

Todo:

We're trivially rewriting the stereographic projection for like the 2nd or 3rd time here. We should add an Eigen template for this instead.

References xrt::tracking::hand::mercury::back_project_keypoint_output(), and t_camera_models_flip_and_project().

Referenced by xrt::tracking::hand::mercury::back_project_keypoint_output(), and xrt::tracking::hand::mercury::HandTracking::cCallbackProcess().

check_outside_view()

static bool xrt::tracking::hand::mercury::check_outside_view ( struct HandTracking *  hgt, struct t_camera_extra_info_one_view  boundary, xrt_vec2 &  keypoint  )

inlinestatic

Todo:

Optimize: Most of this can be calculated once at startup

References xrt::tracking::hand::mercury::check_outside_view().

Referenced by xrt::tracking::hand::mercury::check_outside_view().

getCalibration()

static bool xrt::tracking::hand::mercury::getCalibration ( struct HandTracking *  hgt, t_stereo_camera_calibration &  calibration  )

static

Setup helper functions.

Todo:

Todo:

what are these magic values? they're probably turning the OpenCV formalism into OpenXR, but especially what gives with negating orientation.x?

Todo:

Really? We can totally support cameras with varying resolutions.

References t_camera_model_params_from_t_camera_calibration(), and U_LOGGING_DEBUG.

Referenced by t_hand_tracking_sync_mercury_create().

handle_changed_image_size()

static bool xrt::tracking::hand::mercury::handle_changed_image_size ( HandTracking *  hgt, xrt_size &  new_one_view_size  )

static

Todo:

optimize: can't we just scale camera matrix/etc correctly?

References xrt::tracking::hand::mercury::handle_changed_image_size().

Referenced by xrt::tracking::hand::mercury::HandTracking::cCallbackProcess(), and xrt::tracking::hand::mercury::handle_changed_image_size().

stop_everything_if_hands_are_overlapping()

void xrt::tracking::hand::mercury::stop_everything_if_hands_are_overlapping

(

struct HandTracking *

hgt

)

Todo:

This looks like it sucks, but it doesn't given the current architecture.

Todo:

I really want to try making a discrete optimizer that looks at recent info and decides whether to drop tracking for a hand, switch its handedness or switch to some forthcoming overlapping-hands model. This would likely work by pruning impossible combinations, calculating a loss for each remaining option and picking the least bad one.

References xrt::tracking::hand::mercury::stop_everything_if_hands_are_overlapping().

Referenced by xrt::tracking::hand::mercury::HandTracking::cCallbackProcess(), and xrt::tracking::hand::mercury::stop_everything_if_hands_are_overlapping().

Variable Documentation

valid_flags_ht

enum xrt_space_relation_flags xrt::tracking::hand::mercury::valid_flags_ht

static

Initial value:

= (enum xrt_space_relation_flags)(
    XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT |
    XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT)