From a6c41d568a578b9b1a0df9d110169921e98d9dc0 Mon Sep 17 00:00:00 2001 From: Hanusz Leszek Date: Fri, 19 Aug 2022 14:43:40 +0200 Subject: [PATCH 1/4] Modify tools.clean.py to allow only verification, add GitHub action --- .github/workflows/verify_clean_folder.yml | 20 +++++++++ tools/clean.py | 52 ++++++++++++++++++++--- 2 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/verify_clean_folder.yml diff --git a/.github/workflows/verify_clean_folder.yml b/.github/workflows/verify_clean_folder.yml new file mode 100644 index 0000000..8dcb24c --- /dev/null +++ b/.github/workflows/verify_clean_folder.yml @@ -0,0 +1,20 @@ +name: Verify clean folder + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: 3.10 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install nbdev==1.2.11 + - name: Run tools/clean.py + run: python tools/clean.py --verify-only diff --git a/tools/clean.py b/tools/clean.py index 298e6b2..b23ab83 100755 --- a/tools/clean.py +++ b/tools/clean.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +import argparse +import sys import nbformat from nbdev.export import * from nbdev.clean import * @@ -24,18 +26,58 @@ def clean_tags(cell): cell["source"] = re.sub(r'#\s*' + attr + r'.*?($|\n)', '', cell["source"]) return cell -def proc_nb(fname, dest): +def proc_nb(fname, dest_path, verify_only=False): + """Create a cleaned version of the notebook in fname in the dest_path folder. + + return True if the current file in the dest folder needs to be modified. + """ nb = read_nb(fname) i = get_stop_idx(nb['cells']) nb['cells'] = [clean_tags(c) for j,c in enumerate(nb['cells']) if c['cell_type']=='code' or is_header_cell(c) or is_clean_cell(c) or j >= i] clean_nb(nb, clear_all=True) - with open(dest/fname.name, 'w') as f: nbformat.write(nb, f, version=4) -def proc_all(path='.', dest_path='clean'): + clean_dest = dest_path/fname.name + + try: + existing_nb_clean = read_nb(clean_dest) + except FileNotFoundError: + existing_nb_clean = None + + if nb == existing_nb_clean: + return False + else: + print (f'{clean_dest} is not up to date!') + if not verify_only: + print (f' ==> Modifying {clean_dest}.') + with open(clean_dest, 'w') as f: + nbformat.write(nb, f, version=4) + return True + +def proc_all(path='.', dest_path='clean', verify_only=False): + """Process all the notebooks to create cleaned version in the dest_path folder. + + return True if a modification is needed. + """ path,dest_path = Path(path),Path(dest_path) fns = [f for f in path.iterdir() if f.suffix == '.ipynb' and not f.name.startswith('_')] - for fn in fns: proc_nb(fn, dest=dest_path) + need_cleaning = [proc_nb(fn, dest_path=dest_path, verify_only=verify_only) for fn in fns] -if __name__=='__main__': proc_all() + return any(need_cleaning) +if __name__=='__main__': + + parser = argparse.ArgumentParser(description='Create clean versions of the notebooks, without the prose.') + + parser.add_argument( + '--verify-only', + dest='verify_only', + action='store_true', + help='Only verify if the clean folder is up to date. Used for CI.', + ) + + args = parser.parse_args() + + exit_code = 1 if proc_all(verify_only=args.verify_only) else 0 + + sys.exit(exit_code) From cfff090fba549b25a1e281b1ad71b5f3aa62ae5b Mon Sep 17 00:00:00 2001 From: Hanusz Leszek Date: Fri, 19 Aug 2022 14:47:22 +0200 Subject: [PATCH 2/4] Set Python version to 3.8 --- .github/workflows/verify_clean_folder.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_clean_folder.yml b/.github/workflows/verify_clean_folder.yml index 8dcb24c..1d62fe2 100644 --- a/.github/workflows/verify_clean_folder.yml +++ b/.github/workflows/verify_clean_folder.yml @@ -8,10 +8,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: 3.8 - name: Install dependencies run: | python -m pip install --upgrade pip From ec46e153d9336b2010f2b6ce23fbca9ccca5577e Mon Sep 17 00:00:00 2001 From: Hanusz Leszek Date: Fri, 19 Aug 2022 14:49:39 +0200 Subject: [PATCH 3/4] Test with unclean file --- clean/04_mnist_basics.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean/04_mnist_basics.ipynb b/clean/04_mnist_basics.ipynb index a75eb39..6cc0bbf 100644 --- a/clean/04_mnist_basics.ipynb +++ b/clean/04_mnist_basics.ipynb @@ -174,7 +174,7 @@ "metadata": {}, "outputs": [], "source": [ - "stacked_sevens = torch.stack(seven_tensors).float()/255\n", + "stacked_sevens = torch.stack(seven_tensors)/255\n", "stacked_threes = torch.stack(three_tensors).float()/255\n", "stacked_threes.shape" ] From d74e1f5fcbcbc9f2bb610f4e45a7d4675aefeea3 Mon Sep 17 00:00:00 2001 From: Hanusz Leszek Date: Fri, 19 Aug 2022 14:51:00 +0200 Subject: [PATCH 4/4] Revert "Test with unclean file" This reverts commit ec46e153d9336b2010f2b6ce23fbca9ccca5577e. --- clean/04_mnist_basics.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean/04_mnist_basics.ipynb b/clean/04_mnist_basics.ipynb index 6cc0bbf..a75eb39 100644 --- a/clean/04_mnist_basics.ipynb +++ b/clean/04_mnist_basics.ipynb @@ -174,7 +174,7 @@ "metadata": {}, "outputs": [], "source": [ - "stacked_sevens = torch.stack(seven_tensors)/255\n", + "stacked_sevens = torch.stack(seven_tensors).float()/255\n", "stacked_threes = torch.stack(three_tensors).float()/255\n", "stacked_threes.shape" ]