[scripts] implement script for showing not marked snippets of sources

Author: jauhien

.gitignore

@@ -1,3 +1,4 @@
 target
 src/missing_llvm_bindings/llvmdeps.rs
 tmp
+__pycache__

COPYRIGHT

@@ -1,6 +1,3 @@
-Copyright 2014 Jauhien Piatlicki
-
-Copyright 2013-2014 The Rust Project Developers
-for files taken from the Rust project (see files headers)
+Copyright 2014-2016 Jauhien Piatlicki
 
 Dual licensed under MIT and Apache-2.0 licenses

scripts/generate-from-template.py

@@ -2,99 +2,28 @@
 # -*- coding: utf-8 -*-
 
 """
-    Script to generate README with pieces of source files
+    Script to generate FILE with pieces of source files
 
-    :copyright: (c) 2015 Jauhien Piatlicki
+    :copyright: (c) 2015-2016 Jauhien Piatlicki
     :license: Apache-2.0 or MIT, see LICENSE-* for details
 """
 
 import argparse
-import re
 import sys
 
-def parse_source(source):
-    snippets = {}
-    active_snippets = set()
-    for line in source:
-        if line[0:3] == "//<":
-            active_snippets.update(line[3:].split())
-        elif line[0:3] == "//>":
-            active_snippets = active_snippets.difference(set(line[3:].split()))
-        else:
-            join = False
-            w = False
-            if line[0:5] == "/*j*/":
-                line = line[5:]
-                join = True
-            elif line[0:6] == "/*jw*/":
-                line = line[6:]
-                join = True
-                w = True
-            if join:
-                line = line.lstrip()
-                if w:
-                    line = " " + line
-
-            for snippet in active_snippets:
-                try:
-                    snippets[snippet].append(line)
-                except KeyError:
-                    snippets[snippet] = [line]
-
-                if join:
-                    try:
-                        snippets[snippet][-2] = snippets[snippet][-2].rstrip('\n')
-                    except IndexError:
-                        pass
-
-    return snippets
-
-def parse_source_file(fname):
-    source = []
-    with open(fname) as f:
-        source = f.readlines()
-    return parse_source(source)
-
-class Snippets(object):
-    __slots__ = ['snippets']
-
-    def __init__(self):
-        self.snippets = {}
-
-    def get(self, fname, name):
-        snippets = {}
-        try:
-            snippets = self.snippets[fname]
-        except KeyError:
-            snippets = parse_source_file(fname)
-            self.snippets[fname] = snippets
-        return snippets[name]
-
-def process_readme(readme):
-    snippets = Snippets()
-    result = []
-    regexp = re.compile('<<<(.*):(.*)>>>')
-    for line in readme:
-        match = regexp.match(line)
-        if match:
-            fname = match.group(1)
-            name = match.group(2)
-            result.extend(snippets.get(fname, name))
-        else:
-            result.append(line)
-    return result
+import snippets
 
 def main():
-    parser = argparse.ArgumentParser("Generate README with code snippets")
-    parser.add_argument('readme_in', help='README template')
-    parser.add_argument('readme_out', help='README to be generated')
+    parser = argparse.ArgumentParser("Generate FILE with code snippets")
+    parser.add_argument('file_in', help='FILE template')
+    parser.add_argument('file_out', help='FILE to be generated')
     args = parser.parse_args()
-    readme_in = []
-    with open(args.readme_in) as f:
-        readme_in = f.readlines()
-    readme = process_readme(readme_in)
-    with open(args.readme_out, 'w') as f:
-        f.write("".join(readme))
+    file_in = []
+    with open(args.file_in) as f:
+        file_in = f.readlines()
+    text = snippets.process(file_in)
+    with open(args.file_out, 'w') as f:
+        f.write("".join(text))
 
 if __name__ == "__main__":
     sys.exit(main())

scripts/show-all-not-marked.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Call this script from the root of the repo
+for i in src/*.rs grammar.ebnf;
+do
+    echo
+    echo "********************************************************************************"
+    echo $i
+    echo "********************************************************************************"
+    echo
+    python3 scripts/show-not-marked.py $i
+done

scripts/show-not-marked.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+"""
+    Script to show not marked pieces of file
+
+    :copyright: (c) 2016 Jauhien Piatlicki
+    :license: Apache-2.0 or MIT, see LICENSE-* for details
+"""
+
+import argparse
+import sys
+
+import snippets
+
+def main():
+    parser = argparse.ArgumentParser("Show not marked pieces of FILE")
+    parser.add_argument('file_in', help='input FILE')
+    args = parser.parse_args()
+    file_in = []
+    with open(args.file_in) as f:
+        file_in = f.readlines()
+    file_snippets = snippets.parse_source(file_in)
+    try:
+        for line in file_snippets["not-marked"]:
+            print(line)
+    except KeyError:
+        pass
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/snippets.py

@@ -0,0 +1,89 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+"""
+    Module to work with marked code snippets
+
+    :copyright: (c) 2015-2016 Jauhien Piatlicki
+    :license: Apache-2.0 or MIT, see LICENSE-* for details
+"""
+
+import re
+
+def parse_source(source):
+    snippets = {}
+    active_snippets = set()
+    for line in source:
+        if line[0:3] == "//<":
+            active_snippets.update(line[3:].split())
+        elif line[0:3] == "//>":
+            active_snippets = active_snippets.difference(set(line[3:].split()))
+        else:
+            join = False
+            w = False
+            if line[0:5] == "/*j*/":
+                line = line[5:]
+                join = True
+            elif line[0:6] == "/*jw*/":
+                line = line[6:]
+                join = True
+                w = True
+            if join:
+                line = line.lstrip()
+                if w:
+                    line = " " + line
+
+            if not active_snippets:
+                try:
+                    snippets["not-marked"].append(line)
+                except KeyError:
+                    snippets["not-marked"] = [line]
+            else:
+                for snippet in active_snippets:
+                    try:
+                        snippets[snippet].append(line)
+                    except KeyError:
+                        snippets[snippet] = [line]
+
+                    if join:
+                        try:
+                            snippets[snippet][-2] = snippets[snippet][-2].rstrip('\n')
+                        except IndexError:
+                            pass
+
+    return snippets
+
+def parse_source_file(fname):
+    source = []
+    with open(fname) as f:
+        source = f.readlines()
+    return parse_source(source)
+
+class Snippets(object):
+    __slots__ = ['snippets']
+
+    def __init__(self):
+        self.snippets = {}
+
+    def get(self, fname, name):
+        snippets = {}
+        try:
+            snippets = self.snippets[fname]
+        except KeyError:
+            snippets = parse_source_file(fname)
+            self.snippets[fname] = snippets
+        return snippets[name]
+
+def process(text):
+    snippets = Snippets()
+    result = []
+    regexp = re.compile('<<<(.*):(.*)>>>')
+    for line in text:
+        match = regexp.match(line)
+        if match:
+            fname = match.group(1)
+            name = match.group(2)
+            result.extend(snippets.get(fname, name))
+        else:
+            result.append(line)
+    return result