database/TablePopulateFuncsV4.py
def make_trackingdata_table(table, key):
	from database.TablesDefinitionsV4 import Recording, Session, CCM, MazeComponents

	# split = key['session_name'].split("_")
	# if len(split) > 2: raise ValueError
	# else:
	# 	mouse = split[-1]
	# if key['mouse_id'] != mouse: return

	# skip experiments that i'm not interested in 
	experiment = (Session & key).fetch1("experiment_name")
	if experiment in table.experiments_to_skip: 
		# print("Skipping experiment: ", experiment)
		return

	# Get videos and CCM
	try:
		vid = (Recording.FilePaths & key).fetch1("overview_video")
	except:
		# Try alternative names for the recording UID
		alt_uids = [key['recording_uid']+"Overview", key['recording_uid']+"_1Overview"]
		vid = None
		for uid in alt_uids:
			newkey = key.copy()
			newkey['recording_uid'] = uid
			try:
				vid = (Recording.FilePaths & newkey).fetch1("overview_video")
			except:
				continue
	
		if vid is None:
			print("Could not fetch video for: ", key)
			return
	ccm = pd.DataFrame((CCM & key).fetch())

	# load pose data
	pose_file  = (Recording.FilePaths & key).fetch1("overview_pose")
	try:
		try:
			posedata = pd.read_hdf(pose_file)
		except:  # adjust path to new winstor path name
			pathparts = pose_file.split("\\")
			pathparts.insert(1, "swc")
			pose_file = os.path.join(*pathparts)
			posedata = pd.read_hdf(pose_file)
	except:
		print("Could not find {}".format(pose_file))
		return

	# Insert entry into MAIN CLASS for this videofile
	key['camera'] = 'overview' 
	table.insert1(key)

	# Get the scorer name and the name of the bodyparts
	first_frame = posedata.iloc[0]
	bodyparts = first_frame.index.levels[1]
	scorer = first_frame.index.levels[0]

	"""
		Loop over bodyparts and populate Bodypart Part table
	"""
	bp_data = {}
	for bp in bodyparts:
		if bp not in table.bodyparts: continue  # skip unwanted body parts
		# Get XY pose and correct with CCM matrix
		xy = posedata[scorer[0], bp].values[:, :2]
		try:
			corrected_data = correct_tracking_data(xy, ccm['correction_matrix'][0], ccm['top_pad'][0], ccm['side_pad'][0], experiment, key['uid'])
		except:
			raise ValueError("Something went wrong while trying to correct tracking data, are you sure you have the CCM for this recording? {}".format(key))
		corrected_data = pd.DataFrame.from_dict({'x':corrected_data[:, 0], 'y':corrected_data[:, 1]})

		# get speed
		speed = calc_distance_between_points_in_a_vector_2d(corrected_data.values)

		# get direction of movement
		dir_of_mvt = calc_angle_between_points_of_vector(np.vstack([corrected_data['x'], corrected_data['y']]).T)
		dir_of_mvt[np.where(speed == 0)[0]] = np.nan # no dir of mvmt when there is no mvmt

		# Add new vals to df
		corrected_data['speed'], corrected_data['direction_of_mvmt'] = speed, dir_of_mvt

		# remove low likelihood frames
		bp_data[bp] = corrected_data.copy()
		like = posedata[scorer[0], bp].values[:, 2]
		corrected_data[like < .99] = np.nan

		# If bp is body get the position on the maze
		if 'body' in bp:
			# Get position of maze templates - and shelter
			rois = pd.DataFrame((MazeComponents & key).fetch())

			del rois['uid'], rois['session_name'], rois['mouse_id']

			# Calcualate in which ROI the body is at each frame - and distance from the shelter
			corrected_data['roi_at_each_frame'] = get_roi_at_each_frame(experiment, key['recording_uid'], corrected_data, dict(rois))  # ? roi name
			rois_ids = {p:i for i,p in enumerate(rois.keys())}  # assign a numeric value to each ROI
			corrected_data['roi_at_each_frame'] = np.array([rois_ids[r] for r in corrected_data['roi_at_each_frame']])
			
		# Insert into part table
		bpkey = key.copy()
		bpkey['bpname'] = bp
		bpkey['tracking_data'] = corrected_data.values
		bpkey['x'] = corrected_data.x.values
		bpkey['y'] = corrected_data.y.values
		bpkey['likelihood'] = like
		bpkey['speed'] = corrected_data.speed.values
		bpkey['direction_of_mvmt'] = corrected_data.direction_of_mvmt.values

		table.BodyPartData.insert1(bpkey)

	# populate body segments part table
	body_part_data = pd.DataFrame(table.BodyPartData & key)
	for name, (bp1, bp2) in table.skeleton.items():
		segkey = key.copy()
		segkey['segment_name'], segkey['bp1'], segkey['bp2'] = name, bp1, bp2

		# get likelihoods
		l1, l2  = body_part_data.loc[body_part_data.bpname == bp1].likelihood.values[0], body_part_data.loc[body_part_data.bpname == bp2].likelihood.values[0]
		segkey['likelihood'] = np.min(np.vstack([l1, l2]).T, 1)

		# get the tracking data
		bp1, bp2 = bp_data[bp1].values[:, :2], bp_data[bp2].values[:, :2]

		# get length of the body segment
		bone_orientation = np.array(calc_angle_between_vectors_of_points_2d(bp1.T, bp2.T))

		# Get angular velocity
		bone_angvel = np.array(calc_ang_velocity(bone_orientation))

		# remove nans
		nan_frames = np.where(segkey['likelihood'] <.99)[0]
		bp1[nan_frames] = np.nan
		bp2[nan_frames] = np.nan
		bone_orientation[nan_frames] = np.nan
		bone_angvel[nan_frames] = np.nan

		# Store everything in the part table
		segkey['orientation'] = bone_orientation
		segkey['angular_velocity'] = bone_angvel

		table.BodySegmentData.insert1(segkey)

		Loop over bodyparts and populate Bodypart Part table
	
Loop over bodyparts and populate Bodypart Part table
Loop over bodyparts and populate Bodypart Part table
def make_trackingdata_table(table, key):
	from database.TablesDefinitionsV4 import Recording, Session, CCM, MazeComponents
	experiment = (Session & key).fetch1("experiment_name")
	if experiment in table.experiments_to_skip: 
		return
	try:
		vid = (Recording.FilePaths & key).fetch1("overview_video")
	except:
		alt_uids = [key['recording_uid']+"Overview", key['recording_uid']+"_1Overview"]
		vid = None
		for uid in alt_uids:
			newkey = key.copy()
			newkey['recording_uid'] = uid
			try:
				vid = (Recording.FilePaths & newkey).fetch1("overview_video")
			except:
				continue
		if vid is None:
			print("Could not fetch video for: ", key)
			return
	ccm = pd.DataFrame((CCM & key).fetch())
	pose_file  = (Recording.FilePaths & key).fetch1("overview_pose")
	try:
		try:
			posedata = pd.read_hdf(pose_file)
		except:  
			pathparts = pose_file.split("\\")
			pathparts.insert(1, "swc")
			pose_file = os.path.join(*pathparts)
			posedata = pd.read_hdf(pose_file)
	except:
		print("Could not find {}".format(pose_file))
		return
	key['camera'] = 'overview' 
	table.insert1(key)
	first_frame = posedata.iloc[0]
	bodyparts = first_frame.index.levels[1]
	scorer = first_frame.index.levels[0]
	bp_data = {}
	for bp in bodyparts:
		if bp not in table.bodyparts: continue  
		xy = posedata[scorer[0], bp].values[:, :2]
		try:
			corrected_data = correct_tracking_data(xy, ccm['correction_matrix'][0], ccm['top_pad'][0], ccm['side_pad'][0], experiment, key['uid'])
		except:
			raise ValueError("Something went wrong while trying to correct tracking data, are you sure you have the CCM for this recording? {}".format(key))
		corrected_data = pd.DataFrame.from_dict({'x':corrected_data[:, 0], 'y':corrected_data[:, 1]})
		speed = calc_distance_between_points_in_a_vector_2d(corrected_data.values)
		dir_of_mvt = calc_angle_between_points_of_vector(np.vstack([corrected_data['x'], corrected_data['y']]).T)
		dir_of_mvt[np.where(speed == 0)[0]] = np.nan 
		corrected_data['speed'], corrected_data['direction_of_mvmt'] = speed, dir_of_mvt
		bp_data[bp] = corrected_data.copy()
		like = posedata[scorer[0], bp].values[:, 2]
		corrected_data[like < .99] = np.nan
		if 'body' in bp:
			rois = pd.DataFrame((MazeComponents & key).fetch())
			del rois['uid'], rois['session_name'], rois['mouse_id']
			corrected_data['roi_at_each_frame'] = get_roi_at_each_frame(experiment, key['recording_uid'], corrected_data, dict(rois))  
			rois_ids = {p:i for i,p in enumerate(rois.keys())}  
			corrected_data['roi_at_each_frame'] = np.array([rois_ids[r] for r in corrected_data['roi_at_each_frame']])
		bpkey = key.copy()
		bpkey['bpname'] = bp
		bpkey['tracking_data'] = corrected_data.values
		bpkey['x'] = corrected_data.x.values
		bpkey['y'] = corrected_data.y.values
		bpkey['likelihood'] = like
		bpkey['speed'] = corrected_data.speed.values
		bpkey['direction_of_mvmt'] = corrected_data.direction_of_mvmt.values
		table.BodyPartData.insert1(bpkey)
	body_part_data = pd.DataFrame(table.BodyPartData & key)
	for name, (bp1, bp2) in table.skeleton.items():
		segkey = key.copy()
		segkey['segment_name'], segkey['bp1'], segkey['bp2'] = name, bp1, bp2
		l1, l2  = body_part_data.loc[body_part_data.bpname == bp1].likelihood.values[0], body_part_data.loc[body_part_data.bpname == bp2].likelihood.values[0]
		segkey['likelihood'] = np.min(np.vstack([l1, l2]).T, 1)
		bp1, bp2 = bp_data[bp1].values[:, :2], bp_data[bp2].values[:, :2]
		bone_orientation = np.array(calc_angle_between_vectors_of_points_2d(bp1.T, bp2.T))
		bone_angvel = np.array(calc_ang_velocity(bone_orientation))
		nan_frames = np.where(segkey['likelihood'] <.99)[0]
		bp1[nan_frames] = np.nan
		bp2[nan_frames] = np.nan
		bone_orientation[nan_frames] = np.nan
		bone_angvel[nan_frames] = np.nan
		segkey['orientation'] = bone_orientation
		segkey['angular_velocity'] = bone_angvel
		table.BodySegmentData.insert1(segkey)
def make_trackingdata_table ( table , key ) : from database . TablesDefinitionsV4 import Recording , Session , CCM , MazeComponents experiment = ( Session & key ) . fetch1 ( "experiment_name" ) if experiment in table . experiments_to_skip : return try : vid = ( Recording . FilePaths & key ) . fetch1 ( "overview_video" ) except : alt_uids = [ key [ 'recording_uid' ] + "Overview" , key [ 'recording_uid' ] + "_1Overview" ] vid = None for uid in alt_uids : newkey = key . copy ( ) newkey [ 'recording_uid' ] = uid try : vid = ( Recording . FilePaths & newkey ) . fetch1 ( "overview_video" ) except : continue if vid is None : print ( "Could not fetch video for: " , key ) return ccm = pd . DataFrame ( ( CCM & key ) . fetch ( ) ) pose_file = ( Recording . FilePaths & key ) . fetch1 ( "overview_pose" ) try : try : posedata = pd . read_hdf ( pose_file ) except : pathparts = pose_file . split ( "\\" ) pathparts . insert ( 1 , "swc" ) pose_file = os . path . join ( * pathparts ) posedata = pd . read_hdf ( pose_file ) except : print ( "Could not find {}" . format ( pose_file ) ) return key [ 'camera' ] = 'overview' table . insert1 ( key ) first_frame = posedata . iloc [ 0 ] bodyparts = first_frame . index . levels [ 1 ] scorer = first_frame . index . levels [ 0 ] bp_data = { } for bp in bodyparts : if bp not in table . bodyparts : continue xy = posedata [ scorer [ 0 ] , bp ] . values [ : , : 2 ] try : corrected_data = correct_tracking_data ( xy , ccm [ 'correction_matrix' ] [ 0 ] , ccm [ 'top_pad' ] [ 0 ] , ccm [ 'side_pad' ] [ 0 ] , experiment , key [ 'uid' ] ) except : raise ValueError ( "Something went wrong while trying to correct tracking data, are you sure you have the CCM for this recording? {}" . format ( key ) ) corrected_data = pd . DataFrame . from_dict ( { 'x' : corrected_data [ : , 0 ] , 'y' : corrected_data [ : , 1 ] } ) speed = calc_distance_between_points_in_a_vector_2d ( corrected_data . values ) dir_of_mvt = calc_angle_between_points_of_vector ( np . vstack ( [ corrected_data [ 'x' ] , corrected_data [ 'y' ] ] ) . T ) dir_of_mvt [ np . where ( speed == 0 ) [ 0 ] ] = np . nan corrected_data [ 'speed' ] , corrected_data [ 'direction_of_mvmt' ] = speed , dir_of_mvt bp_data [ bp ] = corrected_data . copy ( ) like = posedata [ scorer [ 0 ] , bp ] . values [ : , 2 ] corrected_data [ like < .99 ] = np . nan if 'body' in bp : rois = pd . DataFrame ( ( MazeComponents & key ) . fetch ( ) ) del rois [ 'uid' ] , rois [ 'session_name' ] , rois [ 'mouse_id' ] corrected_data [ 'roi_at_each_frame' ] = get_roi_at_each_frame ( experiment , key [ 'recording_uid' ] , corrected_data , dict ( rois ) ) rois_ids = { p : i for i , p in enumerate ( rois . keys ( ) ) } corrected_data [ 'roi_at_each_frame' ] = np . array ( [ rois_ids [ r ] for r in corrected_data [ 'roi_at_each_frame' ] ] ) bpkey = key . copy ( ) bpkey [ 'bpname' ] = bp bpkey [ 'tracking_data' ] = corrected_data . values bpkey [ 'x' ] = corrected_data . x . values bpkey [ 'y' ] = corrected_data . y . values bpkey [ 'likelihood' ] = like bpkey [ 'speed' ] = corrected_data . speed . values bpkey [ 'direction_of_mvmt' ] = corrected_data . direction_of_mvmt . values table . BodyPartData . insert1 ( bpkey ) body_part_data = pd . DataFrame ( table . BodyPartData & key ) for name , ( bp1 , bp2 ) in table . skeleton . items ( ) : segkey = key . copy ( ) segkey [ 'segment_name' ] , segkey [ 'bp1' ] , segkey [ 'bp2' ] = name , bp1 , bp2 l1 , l2 = body_part_data . loc [ body_part_data . bpname == bp1 ] . likelihood . values [ 0 ] , body_part_data . loc [ body_part_data . bpname == bp2 ] . likelihood . values [ 0 ] segkey [ 'likelihood' ] = np . min ( np . vstack ( [ l1 , l2 ] ) . T , 1 ) bp1 , bp2 = bp_data [ bp1 ] . values [ : , : 2 ] , bp_data [ bp2 ] . values [ : , : 2 ] bone_orientation = np . array ( calc_angle_between_vectors_of_points_2d ( bp1 . T , bp2 . T ) ) bone_angvel = np . array ( calc_ang_velocity ( bone_orientation ) ) nan_frames = np . where ( segkey [ 'likelihood' ] < .99 ) [ 0 ] bp1 [ nan_frames ] = np . nan bp2 [ nan_frames ] = np . nan bone_orientation [ nan_frames ] = np . nan bone_angvel [ nan_frames ] = np . nan segkey [ 'orientation' ] = bone_orientation segkey [ 'angular_velocity' ] = bone_angvel table . BodySegmentData . insert1 ( segkey )
Loop over bodyparts and populate Bodypart Part table
Loop over bodyparts and populate Bodypart Part table
# split = key['session_name'].split("_") # if len(split) > 2: raise ValueError # else: # 	mouse = split[-1] # if key['mouse_id'] != mouse: return # skip experiments that i'm not interested in  # print("Skipping experiment: ", experiment) # Get videos and CCM # Try alternative names for the recording UID # load pose data # adjust path to new winstor path name # Insert entry into MAIN CLASS for this videofile # Get the scorer name and the name of the bodyparts """
		Loop over bodyparts and populate Bodypart Part table
	""" # skip unwanted body parts # Get XY pose and correct with CCM matrix # get speed # get direction of movement # no dir of mvmt when there is no mvmt # Add new vals to df # remove low likelihood frames # If bp is body get the position on the maze # Get position of maze templates - and shelter # Calcualate in which ROI the body is at each frame - and distance from the shelter # ? roi name # assign a numeric value to each ROI # Insert into part table # populate body segments part table # get likelihoods # get the tracking data # get length of the body segment # Get angular velocity # remove nans # Store everything in the part table
table
key