Merge d74e1f5fcb into e8baa81d89

.github/workflows/verify_clean_folder.yml

@@ -0,0 +1,20 @@
+name: Verify clean folder
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install nbdev==1.2.11
+    - name: Run tools/clean.py
+      run: python tools/clean.py --verify-only

tools/clean.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+import argparse
+import sys
 import nbformat
 from nbdev.export import *
 from nbdev.clean import *
@@ -24,18 +26,58 @@ def clean_tags(cell):
         cell["source"] = re.sub(r'#\s*' + attr + r'.*?($|\n)', '', cell["source"])
     return cell
 
-def proc_nb(fname, dest):
+def proc_nb(fname, dest_path, verify_only=False):
+    """Create a cleaned version of the notebook in fname in the dest_path folder.
+
+    return True if the current file in the dest folder needs to be modified.
+    """
     nb = read_nb(fname)
     i = get_stop_idx(nb['cells'])
     nb['cells'] = [clean_tags(c) for j,c in enumerate(nb['cells']) if
                    c['cell_type']=='code' or is_header_cell(c) or is_clean_cell(c) or j >= i]
     clean_nb(nb, clear_all=True)
-    with open(dest/fname.name, 'w') as f: nbformat.write(nb, f, version=4)
 
-def proc_all(path='.', dest_path='clean'):
+    clean_dest = dest_path/fname.name
+
+    try:
+        existing_nb_clean = read_nb(clean_dest)
+    except FileNotFoundError:
+        existing_nb_clean = None
+
+    if nb == existing_nb_clean:
+        return False
+    else:
+        print (f'{clean_dest} is not up to date!')
+        if not verify_only:
+            print (f'  ==> Modifying {clean_dest}.')
+            with open(clean_dest, 'w') as f:
+                nbformat.write(nb, f, version=4)
+        return True
+
+def proc_all(path='.', dest_path='clean', verify_only=False):
+    """Process all the notebooks to create cleaned version in the dest_path folder.
+
+    return True if a modification is needed.
+    """
     path,dest_path = Path(path),Path(dest_path)
     fns = [f for f in path.iterdir() if f.suffix == '.ipynb' and not f.name.startswith('_')]
-    for fn in fns: proc_nb(fn, dest=dest_path)
+    need_cleaning = [proc_nb(fn, dest_path=dest_path, verify_only=verify_only) for fn in fns]
 
-if __name__=='__main__': proc_all()
+    return any(need_cleaning)
 
+if __name__=='__main__':
+
+    parser = argparse.ArgumentParser(description='Create clean versions of the notebooks, without the prose.')
+
+    parser.add_argument(
+        '--verify-only',
+        dest='verify_only',
+        action='store_true',
+        help='Only verify if the clean folder is up to date. Used for CI.',
+    )
+
+    args = parser.parse_args()
+
+    exit_code = 1 if proc_all(verify_only=args.verify_only) else 0
+
+    sys.exit(exit_code)