Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Make folder for evaluation
auxiliaryfunctions.attempttomakefolder(str(cfg["project_path"]+"/evaluation-results/"))
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
datafn,metadatafn=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction=auxiliaryfunctions.LoadMetadata(os.path.join(cfg["project_path"],metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
#change batch size, if it was edited during analysis!
dlc_cfg['batch_size']=1 #in case this was edited for analysis.
#Create folder structure to store results.
evaluationfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetEvaluationFolder(trainFraction,shuffle,cfg)))
auxiliaryfunctions.attempttomakefolder(evaluationfolder,recursive=True)
#path_train_config = modelfolder / 'train' / 'pose_cfg.yaml'
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(str(modelfolder), 'train'))if "index" in fn])
try: #check if any where found?
Snapshots[0]
except IndexError:
#Data=pd.read_hdf(os.path.join(cfg["project_path"],str(trainingsetfolder),'CollectedData_' + cfg["scorer"] + '.h5'),'df_with_missing')
# Get list of body parts to evaluate network for
comparisonbodyparts=auxiliaryfunctions.IntersectionofBodyPartsandOnesGivenbyUser(cfg,comparisonbodyparts)
##################################################
# Load data...
##################################################
trainFraction=cfg["TrainingFraction"][trainingsetindex]
datafn,metadatafn=auxiliaryfunctions.GetDataandMetaDataFilenames(trainingsetfolder,trainFraction,shuffle,cfg)
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
# Load meta data
data, trainIndices, testIndices, trainFraction=auxiliaryfunctions.LoadMetadata(os.path.join(cfg["project_path"],metadatafn))
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
########################### RESCALING (to global scale)
if rescale==True:
scale=dlc_cfg['global_scale']
print("Rescaling Data to ", scale)
Data=pd.read_hdf(os.path.join(cfg["project_path"],str(trainingsetfolder),'CollectedData_' + cfg["scorer"] + '.h5'),'df_with_missing')*scale
else:
scale=1
Data=pd.read_hdf(os.path.join(cfg["project_path"],str(trainingsetfolder),'CollectedData_' + cfg["scorer"] + '.h5'),'df_with_missing')
evaluationfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetEvaluationFolder(trainFraction,shuffle,cfg)))
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(str(modelfolder), 'train'))if "index" in fn])
tf.reset_default_graph()
start_path=os.getcwd() #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
if cropping is not None:
cfg['cropping']=True
cfg['x1'],cfg['x2'],cfg['y1'],cfg['y2']=cropping
print("Overwriting cropping parameters:", cropping)
print("These are used for all videos, but won't be save to the cfg file.")
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn])
except FileNotFoundError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle))
if cfg['snapshotindex'] == 'all':
print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!")
snapshotindex = -1
else:
snapshotindex=cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
vers = (tf.__version__).split('.')
if int(vers[0]) == 1 and int(vers[1]) > 12:
TF = tf.compat.v1
else:
TF = tf
TF.reset_default_graph()
start_path=os.getcwd() #record cwd to return to this directory in the end
cfg = auxiliaryfunctions.read_config(config)
trainFraction = cfg['TrainingFraction'][trainingsetindex]
modelfolder=os.path.join(cfg["project_path"],str(auxiliaryfunctions.GetModelFolder(trainFraction,shuffle,cfg)))
path_test_config = Path(modelfolder) / 'test' / 'pose_cfg.yaml'
try:
dlc_cfg = load_config(str(path_test_config))
except FileNotFoundError:
raise FileNotFoundError("It seems the model for shuffle %s and trainFraction %s does not exist."%(shuffle,trainFraction))
# Check which snapshots are available and sort them by # iterations
try:
Snapshots = np.array([fn.split('.')[0]for fn in os.listdir(os.path.join(modelfolder , 'train'))if "index" in fn])
except FileNotFoundError:
raise FileNotFoundError("Snapshots not found! It seems the dataset for shuffle %s has not been trained/does not exist.\n Please train it before using it to analyze videos.\n Use the function 'train_network' to train the network for shuffle %s."%(shuffle,shuffle))
if cfg['snapshotindex'] == 'all':
print("Snapshotindex is set to 'all' in the config.yaml file. Running video analysis with all snapshots is very costly! Use the function 'evaluate_network' to choose the best the snapshot. For now, changing snapshot index to -1!")
snapshotindex = -1
else:
snapshotindex=cfg['snapshotindex']
increasing_indices = np.argsort([int(m.split('-')[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
def __init__(self, cfg_path):
self.cfg_path = cfg_path
self.cfg = edict(load_config(cfg_path))
self.cfg.batch_size = 1
self.sess, self.inputs, self.outputs = self.setup_frozen_prediction()
def display_dataset():
logging.basicConfig(level=logging.DEBUG)
cfg = load_config()
dataset = dataset_create(cfg)
dataset.set_shuffle(False)
while True:
batch = dataset.next_batch()
for frame_id in range(1):
img = batch[Batch.inputs][frame_id,:,:,:]
img = np.squeeze(img).astype('uint8')
scmap = batch[Batch.part_score_targets][frame_id,:,:,:]
scmap = np.squeeze(scmap)
# scmask = batch[Batch.part_score_weights]
# if scmask.size > 1:
# scmask = np.squeeze(scmask).astype('uint8')
def train(config_yaml,displayiters,saveiters,maxiters,max_to_keep=5,keepdeconvweights=True,allow_growth=False):
start_path=os.getcwd()
os.chdir(str(Path(config_yaml).parents[0])) #switch to folder of config_yaml (for logging)
setup_logging()
cfg = load_config(config_yaml)
if cfg.dataset_type=='default' or cfg.dataset_type=='tensorpack' or cfg.dataset_type=='deterministic':
print("Switching batchsize to 1, as default/tensorpack/deterministic loaders do not support batches >1. Use imgaug loader.")
cfg['batch_size']=1 #in case this was edited for analysis.-
dataset = create_dataset(cfg)
batch_spec = get_batch_spec(cfg)
batch, enqueue_op, placeholders = setup_preloading(batch_spec)
losses = pose_net(cfg).train(batch)
total_loss = losses['total_loss']
for k, t in losses.items():
TF.summary.scalar(k, t)
merged_summaries = TF.summary.merge_all()
if 'snapshot' in Path(cfg.init_weights).stem and keepdeconvweights:
print("Loading already trained DLC with backbone:", cfg.net_type)
def train(config_yaml,displayiters,saveiters,maxiters,max_to_keep=5,keepdeconvweights=True,allow_growth=False):
start_path=os.getcwd()
os.chdir(str(Path(config_yaml).parents[0])) #switch to folder of config_yaml (for logging)
setup_logging()
cfg = load_config(config_yaml)
if cfg.dataset_type=='default' or cfg.dataset_type=='tensorpack' or cfg.dataset_type=='deterministic':
print("Switching batchsize to 1, as default/tensorpack/deterministic loaders do not support batches >1. Use imgaug loader.")
cfg['batch_size']=1 #in case this was edited for analysis.-
dataset = create_dataset(cfg)
batch_spec = get_batch_spec(cfg)
batch, enqueue_op, placeholders = setup_preloading(batch_spec)
losses = pose_net(cfg).train(batch)
total_loss = losses['total_loss']
for k, t in losses.items():
TF.summary.scalar(k, t)
merged_summaries = TF.summary.merge_all()
if 'snapshot' in Path(cfg.init_weights).stem and keepdeconvweights:
print("Loading already trained DLC with backbone:", cfg.net_type)