Skip to content

Commit

Permalink
[MXAPPS-581] Fixes for broken Straight Dope tests. (apache#11923)
Browse files Browse the repository at this point in the history 
* Update relative paths pointing to the data directory to point to the
  correct place in the testing temporary folder.

* Enable the notebooks that were previously broken because of relative
  file paths not pointing to the correct place.

* Move some notebooks we do not plan to test to the whitelist. These
  notebooks are not published in the Straight Dope book.

* Clean-up: Convert print statements to info/warn/error logging
  statements. Add some logging statements for better status.
  • Loading branch information
@vishaalkapoor @szha
vishaalkapoor authored and szha committed Jul 29, 2018
1 parent a454cb5 commit 4d52926
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 79 deletions.
39 changes: 33 additions & 6 deletions tests/nightly/straight_dope/straight_dope_test_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
the notebook. e.g: `export MXNET_TEST_KERNEL=python2`
"""
import io
import logging
import os
import re
import shutil
Expand All @@ -40,20 +41,26 @@
GIT_REPO = 'https://github.com/zackchase/mxnet-the-straight-dope'
KERNEL = os.getenv('MXNET_TEST_KERNEL', None)
NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook')
RELATIVE_DATA_PATH_REGEX = r'\.\.\/data\/' # Regular expression to match the relative data path.

def _test_notebook(notebook, override_epochs=True):
"""Run Jupyter notebook to catch any execution error.
Args:
notebook : string
notebook name in folder/notebook format
epochs : boolean
override_epochs : boolean
whether or not to override the number of epochs to 1
Returns:
True if the notebook runs without warning or error.
"""
# Some notebooks will fail to run without error if we do not override the data path.
_override_data_path(notebook)

if override_epochs:
_override_epochs(notebook)

return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL, temp_dir=NOTEBOOKS_DIR)


Expand All @@ -63,29 +70,49 @@ def _override_epochs(notebook):
Args:
notebook : string
notebook name in folder/notebook format
"""
notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"

# Read the notebook and set epochs to num_epochs
# Read the notebook and set epochs to num_epochs.
with io.open(notebook_path, 'r', encoding='utf-8') as f:
notebook = f.read()

# Set number of epochs to 1
# Set number of epochs to 1.
modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)

# Replace the original notebook with the modified one.
with io.open(notebook_path, 'w', encoding='utf-8') as f:
f.write(modified_notebook)


def _override_data_path(notebook):
"""Overrides the relative path for the data directory to point to the right place. This is
required as we run the notebooks in a different directory hierarchy more suitable for testing.
Args:
notebook : string
notebook name in folder/notebook format
"""
notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"

# Read the notebook and set epochs to num_epochs.
with io.open(notebook_path, 'r', encoding='utf-8') as f:
notebook = f.read()

# Update the location for the data directory.
modified_notebook = re.sub(RELATIVE_DATA_PATH_REGEX, NOTEBOOKS_DIR + '/data/', notebook)

# Replace the original notebook with the modified one.
with io.open(notebook_path, 'w', encoding='utf-8') as f:
f.write(modified_notebook)

def _download_straight_dope_notebooks():
"""Downloads the Straight Dope Notebooks.
Returns:
True if it succeeds in downloading the notebooks without error.
"""
print('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
logging.info('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)

cmd = [GIT_PATH,
Expand All @@ -98,7 +125,7 @@ def _download_straight_dope_notebooks():
if proc.returncode != 0:
err_msg = 'Error downloading Straight Dope notebooks.\n'
err_msg += msg
print(err_msg)
logging.error(err_msg)
return False
return True

Expand Down
94 changes: 24 additions & 70 deletions tests/nightly/straight_dope/test_notebooks_single_gpu.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,19 @@
'chapter01_crashcourse/introduction',
'chapter01_crashcourse/chapter-one-problem-set',
'chapter02_supervised-learning/environment',
'chapter03_deep-neural-networks/kaggle-gluon-kfold',
'chapter07_distributed-learning/multiple-gpus-scratch',
'chapter07_distributed-learning/multiple-gpus-gluon',
'chapter07_distributed-learning/training-with-multiple-machines'
'chapter07_distributed-learning/training-with-multiple-machines',
'chapter12_time-series/intro-forecasting-gluon',
'chapter12_time-series/intro-forecasting-2-gluon',
'chapter13_unsupervised-learning/vae-gluon',
'chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn',
'chapter17_deep-reinforcement-learning/DQN',
'chapter17_deep-reinforcement-learning/DDQN',
'chapter19_graph-neural-networks/Graph-Neural-Networks',
'chapter16_tensor_methods/tensor_basics',
'cheatsheets/kaggle-gluon-kfold'
]


Expand Down Expand Up @@ -91,10 +101,8 @@ def test_linear_regression_scratch(self):
def test_linear_regression_gluon(self):
assert _test_notebook('chapter02_supervised-learning/linear-regression-gluon')

# TODO(vishaalk): There is a relative file path needs to be fixed so that the
# python code can be run from another directory.
#def test_logistic_regression_gluon(self):
# assert _test_notebook('chapter02_supervised-learning/logistic-regression-gluon')
def test_logistic_regression_gluon(self):
assert _test_notebook('chapter02_supervised-learning/logistic-regression-gluon')

def test_softmax_regression_scratch(self):
assert _test_notebook('chapter02_supervised-learning/softmax-regression-scratch')
Expand Down Expand Up @@ -132,9 +140,6 @@ def test_plumbing(self):
def test_custom_layer(self):
assert _test_notebook('chapter03_deep-neural-networks/custom-layer')

#def test_kaggle_gluon_kfold(self):
# assert _test_notebook('chapter03_deep-neural-networks/kaggle-gluon-kfold')

# TODO(vishaalk): Load params and Save params are deprecated warning.
#def test_serialization(self):
# assert _test_notebook('chapter03_deep-neural-networks/serialization')
Expand Down Expand Up @@ -162,20 +167,14 @@ def test_cnn_batch_norm_gluon(self):

# Chapter 5

# TODO(vishaalk): There is a relative file path needs to be fixed so that the
# python code can be run from another directory.
#def test_simple_rnn(self):
# assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn')
def test_simple_rnn(self):
assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn')

# TODO(vishaalk): There is a relative file path needs to be fixed so that the
# python code can be run from another directory.
#def test_lstm_scratch(self):
# assert _test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')
def test_lstm_scratch(self):
assert _test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')

# TODO(vishaalk): There is a relative file path needs to be fixed so that the
# python code can be run from another directory.
#def test_gru_scratch(self):
# assert _test_notebook('chapter05_recurrent-neural-networks/gru-scratch')
def test_gru_scratch(self):
assert _test_notebook('chapter05_recurrent-neural-networks/gru-scratch')

#def test_rnns_gluon(self):
# assert _test_notebook('chapter05_recurrent-neural-networks/rnns-gluon')
Expand Down Expand Up @@ -263,19 +262,6 @@ def test_lds_scratch(self):
#def test_issm_scratch(self):
# assert _test_notebook('chapter12_time-series/issm-scratch')

# TODO(vishaalk): Error: sequential1_batchnorm0_running_mean' has not been initialized
# def test_intro_forecasting_gluon(self):
# assert _test_notebook('chapter12_time-series/intro-forecasting-gluon')

#def test_intro_forecasting_2_gluon(self):
# assert _test_notebook('chapter12_time-series/intro-forecasting-2-gluon')

# Chapter 13

# TODO(vishaalk): Load params and Save params are deprecated warning.
#def test_vae_gluon(self):
# assert _test_notebook('chapter13_unsupervised-learning/vae-gluon')

# Chapter 14

def test_igan_intro(self):
Expand All @@ -287,46 +273,14 @@ def test_dcgan(self):
def test_generative_adversarial_networks(self):
assert _test_notebook('chapter14_generative-adversarial-networks/conditional')

# Chapter 16

# TODO(vishaalk): Checked failed oshape.Size() != dshape.Size()
#def test_tensor_basics(self):
# assert _test_notebook('chapter16_tensor_methods/tensor_basics')

# TODO(vishaalk): Notebook does not appear to be valid JSON.
#def test_pixel2pixel(self):
# assert _test_notebook('chapter14_generative-adversarial-networks/pixel2pixel')

# Chapter 17

# TODO(vishaalk): Requires OpenAI Gym. Also uses deprecated load_params.
#def test_dqn(self):
# assert _test_notebook('chapter17_deep-reinforcement-learning/DQN')

#def test_ddqn(self):
# assert _test_notebook('chapter17_deep-reinforcement-learning/DDQN')

# Chapter 18

#def test_bayes_by_backprop(self):
# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')

#def test_bayes_by_backprop_gluon(self):
# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')

# TODO(vishaalk): AttributeError: 'list' object has no attribute 'keys'
#def test_bayes_by_backprop_rnn(self):
# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn')

# Chapter 19

# TODO(vishaalk): Requires deepchem
#def test_graph_neural_networks(self):
# assert _test_notebook('chapter19_graph-neural-networks/Graph-Neural-Networks')
# Chapter 18

# Cheatsheets
#def test_bayes_by_backprop(self):
# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')

# TODO(vishaalk): There is a relative file path needs to be fixed so that the
# python code can be run from another directory.
#def test_kaggle_gluon_kfold(self):
# assert _test_notebook('cheatsheets/kaggle-gluon-kfold')
#def test_bayes_by_backprop_gluon(self):
# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')
7 changes: 4 additions & 3 deletions tests/utils/notebook_test/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
warning or exception.
"""
import io
import logging
import os
import shutil
import time
Expand Down Expand Up @@ -57,12 +58,12 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir='
-------
Returns true if the workbook runs with no warning or exception.
"""

logging.info("Running notebook '{}'".format(notebook))
notebook_path = os.path.join(*([notebook_dir] + notebook.split('/')))
working_dir = os.path.join(*([temp_dir] + notebook.split('/')))

if no_cache == '1':
print("Cleaning and setting up temp directory '{}'".format(working_dir))
logging.info("Cleaning and setting up temp directory '{}'".format(working_dir))
shutil.rmtree(temp_dir, ignore_errors=True)

errors = []
Expand Down Expand Up @@ -92,6 +93,6 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir='
if "Warning:" in line:
errors.append("Warning:\n" + line)
if len(errors) > 0:
print('\n'.join(errors))
logging.error('\n'.join(errors))
return False
return True

0 comments on commit 4d52926

Please sign in to comment.