Dev branch rollup

.github/workflows/ci-python.yml

@@ -27,4 +27,4 @@ jobs:
           pip install -r python/lib/requirements.txt
       - name: Test with unittest
         run: |
-          PYTHONPATH="$PYTHONPATH:./python:./python/tutorials" python -m unittest discover -v -s ./python
+          PYTHONPATH="$PYTHONPATH:./python:./python/ppt:./python/tutorials" python -m unittest discover -v -s ./python

doc/commands.md

@@ -1,65 +1,72 @@
-# Emacs
-
-| Keystrokes | Command    |
-|---------|---------------|
-| C-x TAB | indent-rigidly
-| M-^     | delete-indentation
-|         | count-matches
-| M-g g   | goto-line
-| C-x r r | copy-rectangle-to-register
-
 # Ack
 
 | Command | Description |
 |---------|-------------|
-| ack --help-types | Examples: --make --cpp --python --cmake
-| ack -g _pattern_ | Find files whose names match _pattern_
-| ack -g _pattern1_ \| ack -x _pattern2_ | Find in files whose names are given in stdin
-| ack "\[^\\x00-\\x7F]" | Find non-ASCII characters
-| ack -g --ignore-ack-defaults --type-set backup:ext:bak --type=backup . | Find .bak files
-| ack -g --ignore-ack-defaults --type-set emacs:match:/~$/ --type=emacs . | Find Emacs backup files
-| ack -g _stuff_ \| xargs rm | Delete stuff ack found
-| ack -l _string1_ \| xargs perl -pi -e 's/_string1_/_string2_/g' | Global search and replace _string1_ with _string2_ (Linux) 
-| ack -l _string1_ \| xargs perl -p -i.bak -e "s/_string1_/_string2_/" | Global search and replace _string1_ with _string2_ (Windows)
-| ack --print0 -l '[ \t]+$' \| xargs -0 -n1 perl -pi -e 's/[ \t]+$//' | Trim trailing whitespace from files (Linux)
-| ack --print0 -l "[ \t]+$" \| xargs -0 -n1 perl -pi.bak -e "s/[ \t]+$//" | Trim trailing whitespace from files (Linux)
+| `ack --help-types` | Examples: `--text` `--make` `--cpp` `--python` `--cmake` |
+| `ack -g` _pattern_ | Find files whose names match _pattern_|
+| `ack -g` _pattern1_ `| ack -x` _pattern2_ | Find in files whose names are given in `stdin` |
+| `ack "[^\x00-\x7F]"` | Find non-ASCII characters|
+| `ack -g --ignore-ack-defaults --type-set backup:ext:bak --type=backup .` | Find `.bak` files |
+| `ack -g --ignore-ack-defaults --type-set emacs:match:/~$/ --type=emacs .` | Find Emacs backup files|
+| `ack -g` _stuff_ `| xargs rm` | Delete stuff `ack` found |
+| `ack -l` _string1_ `| xargs perl -pi -e 's/`_string1_`/`_string2_`/g'` | Global search and replace _string1_ with _string2_ (Linux) |
+| `ack -l` _string1_ `| xargs perl -p -i.bak -e "s/`_string1_`/`_string2_`/"` | Global search and replace _string1_ with _string2_ (Windows)|
+| `ack --print0 -l '[ \t]+$' | xargs -0 -n1 perl -pi -e 's/[ \t]+$//'` | Trim trailing whitespace from files (Linux)|
+| `ack --print0 -l "[ \t]+$" | xargs -0 -n1 perl -pi.bak -e "s/[ \t]+$//"` | Trim trailing whitespace from files (Linux)|
+
+# Emacs
+
+| Keystrokes | Command                    |
+| ---------- | -------------------------- |
+| `C-x TAB`  | indent-rigidly             |
+| `M-^`      | delete-indentation         |
+|            | count-matches              |
+| `M-g g`    | goto-line                  |
+| `C-x r r`  | copy-rectangle-to-register |
+
+# find/grep
+
+| Command                                                  | Description                                                  |
+| -------------------------------------------------------- | ------------------------------------------------------------ |
+| `find . -name "*.txt" -print0 | xargs -0 grep "\bthe\b"` | Approximately equivalent to `ack --type=text "\bthe\b"`<br />***Warning:*** Gnu `grep` regular express syntax differs from the PCRE implementation that `ack` uses. |
+
+# Linux
+
+| Command                                                      | Description                                                  |
+| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| `diff -rq` _dir1_ _dir2_                                     | Recursively compare two directories                          |
+| `tar cvf - ./`_dir_  `| gzip > `_dir_`.tar.gz`               | `tar` and `gzip` directory                                   |
+| `gunzip --stdout _filename_.tar.gz `| tar tvf -`             | `t` - table of contents                                      |
+| `gunzip --stdout` _filename_`.tar.gz | tar xvf -`            | `x` - extract contents                                       |
+| `find . ! -newermt "`_YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_`"`      | Find files whose modification time is newer than _YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_" |
+| `dd if=/dev/random count=1 bs=16 status=none | hexdump -ve '/1 "%04x"'` | Generate 1 block of 16 random bytes and print as hex ASCII   |
+| `cat` _filename_ `| (read -r; printf "%s\n" "$REPLY"; sort -t"," -k2)` | Sort a CSV file by its 2nd column, keeping the header record as the first record |
+| `xdg-open` _filename_or_url_                                 | Opens the file or URL in the preferred application           |
+| `lowriter --headless --convert-to odt *.docx`                | Convert all Microsoft Word files to LibreOffice format       |
+| `ifconfig | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'` | Print IP address                                             |
 
 # Perl
 
 | Command | Description|
 |---------|------------|
-| perl -wnl -e "/_regex_/ and print;" _filename_ | Use \\x22 to represent double quotes in _regex_
-| _something_ \| perl -wnl -e "/_regex_/ and print;" | Search for _regex_ in stdin
+| `perl -wnl -e "/`_regex_`/ and print;"` _filename_ | If you need to search for double quotes, use `\x22` in _regex_ |
+| _something_ `| perl -wnl -e "/`_regex_`/ and print;"` | Search for _regex_ in `stdin` |
 
 # Python
 
 | Command | Description|
 |---------|------------|
-| python -c "import sys; print('\\n'.join(sys.path))"| Print PYTHONPATH
+| `python -c "import sys; print('\\n'.join(sys.path))"` | Print PYTHONPATH|
 
 # Windows
 
 ```xargs``` implementation for Windows:
+
   * https://helloacm.com/simple-xargs-batch-implementation-for-windows/
   * https://github.com/DoctorLai/BatchUtils/blob/master/xargs.cmd
 
 Quick and dirty file find: 
-  * DIR /S /B *.h *.cpp
+  * `DIR /S /B *.h *.cpp`
 
-Add shortcuts to Windows Explorer RMB->Send To context menu:
-  * C:\\Users\_username_\\AppData\\Roaming\\Microsoft\\Windows\\SendTo
-
-# Linux
-
-| Command | Description
-|---------|-------------|
-| diff -rq _dir1_ _dir2_ | Recursively compare two directories
-| tar cvf - ./_dir_ \| gzip > _dir_.tar.gz | tar and gzip directory
-| gunzip --stdout _filename_.tar.gz \| tar tvf - | 't' - table of contents
-| gunzip --stdout _filename_.tar.gz \| tar xvf - | 'x' - extract contents
-| find . ! -newermt "_YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_"| Find files whose modification time is newer than _YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_"
-| dd if=/dev/random count=1 bs=16 status=none \| hexdump -ve '/1 "%04x"'| Generate 1 block of 16 random bytes and print as hex ASCII
-| cat _filename_.csv \| (read -r; printf "%s\n" "$REPLY"; sort -t"," -k2) > _filename_-sorted.csv | Sort a CSV file by its 2nd column, keeping the header record as the first record.
-| xdg-open _filename_or_url_ | Opens the file or URL in the preferred application.
-| lowriter --headless --convert-to odt *.docx | Convert all Microsoft Word files to LibreOffice format.
-| ifconfig \| grep "inet " \| grep -Fv 127.0.0.1 \| awk '{print $2}' | Print IP address.
+Add shortcuts to Windows Explorer `RMB`->`Send To` context menu:
+  * `C:\Users\_username_\AppData\Roaming\Microsoft\Windows\SendTo`

python/ppt/diff.py

@@ -214,7 +214,7 @@ def main(options):
     return 0
 
 
-if __name__ == '__main__':
+def get_parser():
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
@@ -264,6 +264,11 @@ def main(options):
         help='Names of files or directories to compare.',
         nargs=2)
 
+    return parser
+
+
+if __name__ == '__main__':
+    parser = get_parser()
     options = parser.parse_args()
 
     sys.exit(main(options))

python/ppt/tests/data/bar.txt

@@ -0,0 +1,4 @@
+abcde
+zzzzz
+klmnoy
+tuvwxy

python/ppt/tests/data/file1.txt

@@ -0,0 +1,24 @@
+Now we are engaged in a great civil war, testing whether that nation,
+or any nation so conceived and so dedicated, can long endure. We are
+met on a great battle-field of that war. We have come to dedicate a
+portion of that field, as a final resting place for those who here
+gave their lives that that nation might live. It is altogether
+fitting and proper that we should do this.
+
+But, in a larger sense, we can not dedicate -- we can not consecrate
+-- we can not hallow -- this ground. The brave men, living and dead,
+who struggled here, have consecrated it, far above our poor power to
+add or detract. The world will little note, nor long remember what we
+say here, but it can never forget what they did here. It is for us the
+living, rather, to be dedicated here to the unfinished work which they
+who fought here have thus far so nobly advanced. It is rather for us
+to be here dedicated to the great task remaining before us -- that
+from these honored dead we take increased devotion to that cause for
+which they gave the last full measure of devotion -- that we here
+highly resolve that these dead shall not have died in vain -- that
+this nation, under God, shall have a new birth of freedom -- and that
+government of the people, by the people, for the people, shall not
+perish from the earth.
+
+Abraham Lincoln
+November 19, 1863

python/ppt/tests/data/file2.txt

@@ -0,0 +1,26 @@
+Now we are engaged in a great civil war, testing whether that nation,
+or any nation so conceived and so dedicated, can long endure. We are
+met on a great battle-field of that war. We have come to dedicate a
+portion of that field, as a final resting place for those who here
+gave their lives that that nation might live. It is altogether
+fitting and proper that we should do this.
+
+We also sometimes add a line to a file.
+
+But, in a larger sense, we can not dedicate -- we can not consecrate
+-- we can not hallow -- this ground. The brave men, living and dead,
+who struggled here, have consecrated it, far above our poor power to
+add or detract. The world will little note, nor long remember what we
+say here, but it can never forget what they did here. It is for us the
+living, rather, to be dedicated here to the unfinished work which they
+who fought here have thus far so nobly advanced. It is rather for us
+to be here dedicated to the great task remaining before us -- that
+from these honored dead we take increased devotion to that cause for
+which they gave the last full measure of devotion -- that we here
+highly resolve that these dead shall not have died in vain -- that
+this nation, under God, shall have a new birth of freedom -- and that
+government of the people, by the people, for the people, shall not
+perish from the earth because we changed a line in the file.
+
+Abraham Lincoln
+November 19, 1863

python/ppt/tests/data/foo.txt

@@ -0,0 +1,3 @@
+abcde
+fgjij
+klmno

python/ppt/tests/data/gettysburg_address.txt

@@ -0,0 +1,25 @@
+Four score and seven years ago our fathers brought forth on this
+continent a new nation, conceived in Liberty, and dedicated to the
+proposition that all men are created equal.
+
+Now we are engaged in a great civil war, testing whether that nation
+or any nation so conceived and so dedicated, can long endure. We are
+met on a great battle-field of that war. We have come to dedicate a
+portion of that field, as a final resting place for those who here
+gave their lives that that nation might live. It is altogether fitting
+and proper that we should do this.
+
+But, in a larger sense, we can not dedicate--we can not consecrate--we
+can not hallow--this ground. The brave men, living and dead, who
+struggled here, have consecrated it, far above our poor power to add
+or detract. The world will little note, nor long remember what we say
+here, but it can never forget what they did here. It is for us the
+living, rather, to be dedicated here to the unfinished work which they
+who fought here have thus far so nobly advanced. It is rather for us
+to be here dedicated to the great task remaining before us—that from
+these honored dead we take increased devotion to that cause for which
+they gave the last full measure of devotion—that we here highly
+resolve that these dead shall not have died in vain—that this nation,
+under God, shall have a new birth of freedom—and that government of
+the people, by the people, for the people, shall not perish from the
+earth.

python/ppt/tests/testdiff.py

@@ -0,0 +1,134 @@
+""" testdiff.py - Unit test diff.py. """
+
+import ppt.diff
+import unittest
+import warnings
+import os
+import io
+from contextlib import redirect_stdout, redirect_stderr
+
+# Locate the data folder relative to this test file.
+TEST_DIR = os.path.dirname(__file__)
+DATA_DIR = os.path.join(TEST_DIR, 'data')
+
+
+""" mkfile - Utility to populate a file. """
+def mkfile(filename, body=None):
+    with open(filename, 'w') as f:
+        f.write(body or filename)
+    return
+
+
+""" make_example_dir - Utility to create a directory hierarchy of files.  """
+def make_example_dir(top):
+    if not os.path.exists(top):
+        os.mkdir(top)
+    curdir = os.getcwd()
+    os.chdir(top)
+
+    os.mkdir('dir1')
+    os.mkdir('dir2')
+
+    mkfile('dir1/file_only_in_dir1')
+    mkfile('dir2/file_only_in_dir2')
+
+    os.mkdir('dir1/dir_only_in_dir1')
+    os.mkdir('dir2/dir_only_in_dir2')
+
+    os.mkdir('dir1/common_dir')
+    os.mkdir('dir2/common_dir')
+
+    mkfile('dir1/common_file', 'this file is the same')
+    mkfile('dir2/common_file', 'this file is the same')
+
+    mkfile('dir1/not_the_same', 'aaaa bbbb cccc\n')
+    mkfile('dir2/not_the_same', 'dddd eeee ffff\n')
+
+    mkfile('dir1/file_in_dir1', 'This is a file in dir1')
+    os.mkdir('dir2/file_in_dir1')
+
+    os.chdir(curdir)
+
+
+class TestDiff(unittest.TestCase):
+    def test_diff_two_files(self):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ResourceWarning)
+            parser = ppt.diff.get_parser()
+            options = parser.parse_args([ os.path.join(DATA_DIR, 'foo.txt'), os.path.join(DATA_DIR, 'bar.txt') ])
+
+            # Capture stdout when running.
+            out = io.StringIO()
+            with redirect_stdout(out):
+                ppt.diff.main(options)
+
+            # Compare the results.
+            captured_stdout = out.getvalue()
+            expected_result = """2,3c2,4
+< fgjij
+< klmno
+---
+> zzzzz
+> klmnoy
+> tuvwxy
+"""
+            self.assertEqual(captured_stdout, expected_result)
+
+    def test_diff_another_two_files(self):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ResourceWarning)
+            parser = ppt.diff.get_parser()
+            options = parser.parse_args([ os.path.join(DATA_DIR, 'file1.txt'), os.path.join(DATA_DIR, 'file2.txt') ])
+
+            # Capture stdout when running.
+            out = io.StringIO()
+            with redirect_stdout(out):
+                ppt.diff.main(options)
+
+            # Compare the results.
+            captured_stdout = out.getvalue()
+            expected_result = """6a7,8
+> 
+> We also sometimes add a line to a file.
+21c23
+< perish from the earth.
+---
+> perish from the earth because we changed a line in the file.
+"""
+            self.assertEqual(captured_stdout, expected_result)
+
+    def test_diff_recursively_compare_two_directory_trees(self):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ResourceWarning)
+
+            # Make the directory hierarchies (dir1 + dir2)
+            make_example_dir(os.path.join(DATA_DIR, 'example'))
+            make_example_dir(os.path.join(DATA_DIR, 'example/dir1/common_dir'))
+            make_example_dir(os.path.join(DATA_DIR, 'example/dir2/common_dir'))
+
+            parser = ppt.diff.get_parser()
+            options = parser.parse_args([ '-r', os.path.join(DATA_DIR, 'example/dir1'), os.path.join(DATA_DIR, 'example/dir2') ])
+
+            # Capture stdout when running.
+            out = io.StringIO()
+            with redirect_stdout(out):
+                ppt.diff.main(options)
+
+            # Compare the results.
+            captured_stdout = out.getvalue()
+            s = captured_stdout.split('\n')
+
+            self.assertRegex(s[0], r'Only in .+/data/example/dir1:$')
+            self.assertRegex(s[1], r'dir_only_in_dir1')
+            self.assertRegex(s[2], r'file_only_in_dir1')
+            self.assertRegex(s[3], r'Only in .+/data/example/dir2:$')
+            self.assertRegex(s[4], r'dir_only_in_dir2')
+            self.assertRegex(s[5], r'file_only_in_dir2')
+            self.assertRegex(s[6], r'diff .+/data/example/dir1/not_the_same .+/data/example/dir2/not_the_same$')
+
+            diffs = '\n'.join(s[7:11])
+            expected_diffs = """1c1
+< aaaa bbbb cccc
+---
+> dddd eeee ffff"""
+            self.assertEqual(diffs, expected_diffs)