From b476d0285024132ab21b3dd39c2ce39c9d72c15e Mon Sep 17 00:00:00 2001
From: Bar <elhararb@gmail.com>
Date: Fri, 15 Feb 2019 00:52:11 +0200
Subject: [PATCH] Store config files in logdir/configs directory (#156)
Modified log_execution_env_state() to store
configuration file in the output directory,
under 'configs' sub-directory it creates.
At this time, the only configuration file is
passed via args.compress
---
apputils/execution_env.py | 34 ++++++++++++++-----
.../compress_classifier.py | 6 ++--
2 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/apputils/execution_env.py b/apputils/execution_env.py
index 1350650..9b70946 100755
--- a/apputils/execution_env.py
+++ b/apputils/execution_env.py
@@ -20,32 +20,36 @@ This is helpful if you want to recreate an experiment at a later time, or if
you want to understand the environment in which you execute the training.
"""
-import sys
-import os
-import time
-import platform
+import contextlib
import logging
import logging.config
+import os
+import platform
+import shutil
+import sys
+import time
+
+from git import Repo, InvalidGitRepositoryError
import numpy as np
import torch
-from git import Repo, InvalidGitRepositoryError
-HAVE_LSB = True
try:
import lsb_release
+ HAVE_LSB = True
except ImportError:
HAVE_LSB = False
logger = logging.getLogger("app_cfg")
-def log_execution_env_state(app_args, gitroot='.'):
+def log_execution_env_state(config_filename=None, logdir=None, gitroot='.'):
"""Log information about the execution environment.
It is recommeneded to log this information so it can be used for referencing
at a later time.
Args:
- app_args (dict): the command line arguments passed to the application
+ config_filename: filename to store in logdir, if logdir is set
+ logdir: log directory
git_root: the path to the .git root directory
"""
@@ -81,7 +85,19 @@ def log_execution_env_state(app_args, gitroot='.'):
logger.debug("PyTorch: %s", torch.__version__)
logger.debug("Numpy: %s", np.__version__)
log_git_state()
- logger.debug("Command line: %s", " ".join(app_args))
+ logger.debug("Command line: %s", " ".join(sys.argv))
+
+ if (logdir is None) or (config_filename is None):
+ return
+ # clone configuration files to output directory
+ configs_dest = os.path.join(logdir, 'configs')
+ with contextlib.suppress(FileExistsError):
+ os.makedirs(configs_dest)
+ if os.path.exists(os.path.join(configs_dest, config_filename)):
+ logger.debug('{} already exists in logdir'.format(
+ os.path.basename(config_filename) or config_filename))
+ else:
+ shutil.copy(config_filename, configs_dest)
def config_pylogger(log_cfg_file, experiment_name, output_dir='logs'):
diff --git a/examples/classifier_compression/compress_classifier.py b/examples/classifier_compression/compress_classifier.py
index b3d9513..29412f1 100755
--- a/examples/classifier_compression/compress_classifier.py
+++ b/examples/classifier_compression/compress_classifier.py
@@ -90,8 +90,7 @@ def main():
global msglogger
# Parse arguments
- prsr = parser.get_parser()
- args = prsr.parse_args()
+ args = parser.get_parser().parse_args()
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
@@ -99,7 +98,8 @@ def main():
# Log various details about the execution environment. It is sometimes useful
# to refer to past experiment executions and this information may be useful.
- apputils.log_execution_env_state(sys.argv, gitroot=module_path)
+ apputils.log_execution_env_state(args.compress,
+ msglogger.logdir, gitroot=module_path)
msglogger.debug("Distiller: %s", distiller.__version__)
start_epoch = 0
--
GitLab