Skip to content
Snippets Groups Projects
Commit b476d028 authored by Bar's avatar Bar Committed by Neta Zmora
Browse files

Store config files in logdir/configs directory (#156)

Modified log_execution_env_state() to store
configuration file in the output directory,
under 'configs' sub-directory it creates.

At this time, the only configuration file is
passed via args.compress
parent ac9f61c0
No related branches found
No related tags found
No related merge requests found
...@@ -20,32 +20,36 @@ This is helpful if you want to recreate an experiment at a later time, or if ...@@ -20,32 +20,36 @@ This is helpful if you want to recreate an experiment at a later time, or if
you want to understand the environment in which you execute the training. you want to understand the environment in which you execute the training.
""" """
import sys import contextlib
import os
import time
import platform
import logging import logging
import logging.config import logging.config
import os
import platform
import shutil
import sys
import time
from git import Repo, InvalidGitRepositoryError
import numpy as np import numpy as np
import torch import torch
from git import Repo, InvalidGitRepositoryError
HAVE_LSB = True
try: try:
import lsb_release import lsb_release
HAVE_LSB = True
except ImportError: except ImportError:
HAVE_LSB = False HAVE_LSB = False
logger = logging.getLogger("app_cfg") logger = logging.getLogger("app_cfg")
def log_execution_env_state(app_args, gitroot='.'): def log_execution_env_state(config_filename=None, logdir=None, gitroot='.'):
"""Log information about the execution environment. """Log information about the execution environment.
It is recommeneded to log this information so it can be used for referencing It is recommeneded to log this information so it can be used for referencing
at a later time. at a later time.
Args: Args:
app_args (dict): the command line arguments passed to the application config_filename: filename to store in logdir, if logdir is set
logdir: log directory
git_root: the path to the .git root directory git_root: the path to the .git root directory
""" """
...@@ -81,7 +85,19 @@ def log_execution_env_state(app_args, gitroot='.'): ...@@ -81,7 +85,19 @@ def log_execution_env_state(app_args, gitroot='.'):
logger.debug("PyTorch: %s", torch.__version__) logger.debug("PyTorch: %s", torch.__version__)
logger.debug("Numpy: %s", np.__version__) logger.debug("Numpy: %s", np.__version__)
log_git_state() log_git_state()
logger.debug("Command line: %s", " ".join(app_args)) logger.debug("Command line: %s", " ".join(sys.argv))
if (logdir is None) or (config_filename is None):
return
# clone configuration files to output directory
configs_dest = os.path.join(logdir, 'configs')
with contextlib.suppress(FileExistsError):
os.makedirs(configs_dest)
if os.path.exists(os.path.join(configs_dest, config_filename)):
logger.debug('{} already exists in logdir'.format(
os.path.basename(config_filename) or config_filename))
else:
shutil.copy(config_filename, configs_dest)
def config_pylogger(log_cfg_file, experiment_name, output_dir='logs'): def config_pylogger(log_cfg_file, experiment_name, output_dir='logs'):
......
...@@ -90,8 +90,7 @@ def main(): ...@@ -90,8 +90,7 @@ def main():
global msglogger global msglogger
# Parse arguments # Parse arguments
prsr = parser.get_parser() args = parser.get_parser().parse_args()
args = prsr.parse_args()
if not os.path.exists(args.output_dir): if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir) os.makedirs(args.output_dir)
...@@ -99,7 +98,8 @@ def main(): ...@@ -99,7 +98,8 @@ def main():
# Log various details about the execution environment. It is sometimes useful # Log various details about the execution environment. It is sometimes useful
# to refer to past experiment executions and this information may be useful. # to refer to past experiment executions and this information may be useful.
apputils.log_execution_env_state(sys.argv, gitroot=module_path) apputils.log_execution_env_state(args.compress,
msglogger.logdir, gitroot=module_path)
msglogger.debug("Distiller: %s", distiller.__version__) msglogger.debug("Distiller: %s", distiller.__version__)
start_epoch = 0 start_epoch = 0
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment