diff --git a/distiller/apputils/execution_env.py b/distiller/apputils/execution_env.py index 0c59745fba14d8fdd4ece610177c8213ade70df7..34cfafe8d1f3a18c5d2414202514b1948af509ce 100755 --- a/distiller/apputils/execution_env.py +++ b/distiller/apputils/execution_env.py @@ -20,7 +20,6 @@ This is helpful if you want to recreate an experiment at a later time, or if you want to understand the environment in which you execute the training. """ -import contextlib import logging import logging.config import os @@ -41,16 +40,16 @@ except ImportError: logger = logging.getLogger("app_cfg") -def log_execution_env_state(config_path=None, logdir=None, gitroot='.'): +def log_execution_env_state(config_paths=None, logdir=None, gitroot='.'): """Log information about the execution environment. - File 'config_path' will be copied to directory 'logdir'. A common use-case + Files in 'config_paths' will be copied to directory 'logdir'. A common use-case is passing the path to a (compression) schedule YAML file. Storing a copy of the schedule file, with the experiment logs, is useful in order to reproduce experiments. Args: - config_path: path to config file, used only when logdir is set + config_paths: path(s) to config file(s), used only when logdir is set logdir: log directory git_root: the path to the .git root directory """ @@ -89,21 +88,25 @@ def log_execution_env_state(config_path=None, logdir=None, gitroot='.'): log_git_state() logger.debug("Command line: %s", " ".join(sys.argv)) - if (logdir is None) or (config_path is None): + if (logdir is None) or (config_paths is None): return + # clone configuration files to output directory configs_dest = os.path.join(logdir, 'configs') - with contextlib.suppress(FileExistsError): - os.makedirs(configs_dest) - if os.path.exists(os.path.join(configs_dest, - os.path.basename(config_path))): - logger.debug('{} already exists in logdir'.format( - os.path.basename(config_path) or config_path)) - else: - try: - shutil.copy(config_path, configs_dest) - except OSError as e: - logger.debug('Failed to copy of config file: {}'.format(str(e))) + + if isinstance(config_paths, str) or not hasattr(config_paths, '__iter__'): + config_paths = [config_paths] + for cpath in config_paths: + os.makedirs(configs_dest, exist_ok=True) + + if os.path.exists(os.path.join(configs_dest, os.path.basename(cpath))): + logger.debug('{} already exists in logdir'.format( + os.path.basename(cpath) or cpath)) + else: + try: + shutil.copy(cpath, configs_dest) + except OSError as e: + logger.debug('Failed to copy of config file: {}'.format(str(e))) def config_pylogger(log_cfg_file, experiment_name, output_dir='logs'): diff --git a/examples/classifier_compression/compress_classifier.py b/examples/classifier_compression/compress_classifier.py index 883ff9285a9ca68b01438155a1d1cbac12811fa6..4fa93859482649bea9c360ac140f07fb3f338f46 100755 --- a/examples/classifier_compression/compress_classifier.py +++ b/examples/classifier_compression/compress_classifier.py @@ -97,7 +97,9 @@ def main(): # Log various details about the execution environment. It is sometimes useful # to refer to past experiment executions and this information may be useful. - apputils.log_execution_env_state(args.compress, msglogger.logdir, gitroot=module_path) + apputils.log_execution_env_state( + filter(None, [args.compress, args.qe_stats_file]), # remove both None and empty strings + msglogger.logdir, gitroot=module_path) msglogger.debug("Distiller: %s", distiller.__version__) start_epoch = 0