diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index 1cefb05..2d8557e 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -27,4 +27,4 @@ jobs: pip install -r python/lib/requirements.txt - name: Test with unittest run: | - PYTHONPATH="$PYTHONPATH:./python:./python/tutorials" python -m unittest discover -v -s ./python + PYTHONPATH="$PYTHONPATH:./python:./python/ppt:./python/tutorials" python -m unittest discover -v -s ./python diff --git a/doc/commands.md b/doc/commands.md index 82c8173..0794176 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -1,65 +1,72 @@ -# Emacs - -| Keystrokes | Command | -|---------|---------------| -| C-x TAB | indent-rigidly -| M-^ | delete-indentation -| | count-matches -| M-g g | goto-line -| C-x r r | copy-rectangle-to-register - # Ack | Command | Description | |---------|-------------| -| ack --help-types | Examples: --make --cpp --python --cmake -| ack -g _pattern_ | Find files whose names match _pattern_ -| ack -g _pattern1_ \| ack -x _pattern2_ | Find in files whose names are given in stdin -| ack "\[^\\x00-\\x7F]" | Find non-ASCII characters -| ack -g --ignore-ack-defaults --type-set backup:ext:bak --type=backup . | Find .bak files -| ack -g --ignore-ack-defaults --type-set emacs:match:/~$/ --type=emacs . | Find Emacs backup files -| ack -g _stuff_ \| xargs rm | Delete stuff ack found -| ack -l _string1_ \| xargs perl -pi -e 's/_string1_/_string2_/g' | Global search and replace _string1_ with _string2_ (Linux) -| ack -l _string1_ \| xargs perl -p -i.bak -e "s/_string1_/_string2_/" | Global search and replace _string1_ with _string2_ (Windows) -| ack --print0 -l '[ \t]+$' \| xargs -0 -n1 perl -pi -e 's/[ \t]+$//' | Trim trailing whitespace from files (Linux) -| ack --print0 -l "[ \t]+$" \| xargs -0 -n1 perl -pi.bak -e "s/[ \t]+$//" | Trim trailing whitespace from files (Linux) +| `ack --help-types` | Examples: `--text` `--make` `--cpp` `--python` `--cmake` | +| `ack -g` _pattern_ | Find files whose names match _pattern_| +| `ack -g` _pattern1_ `| ack -x` _pattern2_ | Find in files whose names are given in `stdin` | +| `ack "[^\x00-\x7F]"` | Find non-ASCII characters| +| `ack -g --ignore-ack-defaults --type-set backup:ext:bak --type=backup .` | Find `.bak` files | +| `ack -g --ignore-ack-defaults --type-set emacs:match:/~$/ --type=emacs .` | Find Emacs backup files| +| `ack -g` _stuff_ `| xargs rm` | Delete stuff `ack` found | +| `ack -l` _string1_ `| xargs perl -pi -e 's/`_string1_`/`_string2_`/g'` | Global search and replace _string1_ with _string2_ (Linux) | +| `ack -l` _string1_ `| xargs perl -p -i.bak -e "s/`_string1_`/`_string2_`/"` | Global search and replace _string1_ with _string2_ (Windows)| +| `ack --print0 -l '[ \t]+$' | xargs -0 -n1 perl -pi -e 's/[ \t]+$//'` | Trim trailing whitespace from files (Linux)| +| `ack --print0 -l "[ \t]+$" | xargs -0 -n1 perl -pi.bak -e "s/[ \t]+$//"` | Trim trailing whitespace from files (Linux)| + +# Emacs + +| Keystrokes | Command | +| ---------- | -------------------------- | +| `C-x TAB` | indent-rigidly | +| `M-^` | delete-indentation | +| | count-matches | +| `M-g g` | goto-line | +| `C-x r r` | copy-rectangle-to-register | + +# find/grep + +| Command | Description | +| -------------------------------------------------------- | ------------------------------------------------------------ | +| `find . -name "*.txt" -print0 | xargs -0 grep "\bthe\b"` | Approximately equivalent to `ack --type=text "\bthe\b"`
***Warning:*** Gnu `grep` regular express syntax differs from the PCRE implementation that `ack` uses. | + +# Linux + +| Command | Description | +| ------------------------------------------------------------ | ------------------------------------------------------------ | +| `diff -rq` _dir1_ _dir2_ | Recursively compare two directories | +| `tar cvf - ./`_dir_ `| gzip > `_dir_`.tar.gz` | `tar` and `gzip` directory | +| `gunzip --stdout _filename_.tar.gz `| tar tvf -` | `t` - table of contents | +| `gunzip --stdout` _filename_`.tar.gz | tar xvf -` | `x` - extract contents | +| `find . ! -newermt "`_YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_`"` | Find files whose modification time is newer than _YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_" | +| `dd if=/dev/random count=1 bs=16 status=none | hexdump -ve '/1 "%04x"'` | Generate 1 block of 16 random bytes and print as hex ASCII | +| `cat` _filename_ `| (read -r; printf "%s\n" "$REPLY"; sort -t"," -k2)` | Sort a CSV file by its 2nd column, keeping the header record as the first record | +| `xdg-open` _filename_or_url_ | Opens the file or URL in the preferred application | +| `lowriter --headless --convert-to odt *.docx` | Convert all Microsoft Word files to LibreOffice format | +| `ifconfig | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'` | Print IP address | # Perl | Command | Description| |---------|------------| -| perl -wnl -e "/_regex_/ and print;" _filename_ | Use \\x22 to represent double quotes in _regex_ -| _something_ \| perl -wnl -e "/_regex_/ and print;" | Search for _regex_ in stdin +| `perl -wnl -e "/`_regex_`/ and print;"` _filename_ | If you need to search for double quotes, use `\x22` in _regex_ | +| _something_ `| perl -wnl -e "/`_regex_`/ and print;"` | Search for _regex_ in `stdin` | # Python | Command | Description| |---------|------------| -| python -c "import sys; print('\\n'.join(sys.path))"| Print PYTHONPATH +| `python -c "import sys; print('\\n'.join(sys.path))"` | Print PYTHONPATH| # Windows ```xargs``` implementation for Windows: + * https://helloacm.com/simple-xargs-batch-implementation-for-windows/ * https://github.com/DoctorLai/BatchUtils/blob/master/xargs.cmd Quick and dirty file find: - * DIR /S /B *.h *.cpp + * `DIR /S /B *.h *.cpp` -Add shortcuts to Windows Explorer RMB->Send To context menu: - * C:\\Users\_username_\\AppData\\Roaming\\Microsoft\\Windows\\SendTo - -# Linux - -| Command | Description -|---------|-------------| -| diff -rq _dir1_ _dir2_ | Recursively compare two directories -| tar cvf - ./_dir_ \| gzip > _dir_.tar.gz | tar and gzip directory -| gunzip --stdout _filename_.tar.gz \| tar tvf - | 't' - table of contents -| gunzip --stdout _filename_.tar.gz \| tar xvf - | 'x' - extract contents -| find . ! -newermt "_YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_"| Find files whose modification time is newer than _YYYY_-_MM_-_DD_ _HH_:_MM_:_SS_" -| dd if=/dev/random count=1 bs=16 status=none \| hexdump -ve '/1 "%04x"'| Generate 1 block of 16 random bytes and print as hex ASCII -| cat _filename_.csv \| (read -r; printf "%s\n" "$REPLY"; sort -t"," -k2) > _filename_-sorted.csv | Sort a CSV file by its 2nd column, keeping the header record as the first record. -| xdg-open _filename_or_url_ | Opens the file or URL in the preferred application. -| lowriter --headless --convert-to odt *.docx | Convert all Microsoft Word files to LibreOffice format. -| ifconfig \| grep "inet " \| grep -Fv 127.0.0.1 \| awk '{print $2}' | Print IP address. +Add shortcuts to Windows Explorer `RMB`->`Send To` context menu: + * `C:\Users\_username_\AppData\Roaming\Microsoft\Windows\SendTo` diff --git a/python/ppt/__init__.py b/python/ppt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/ppt/diff.py b/python/ppt/diff.py index f50ab06..4434330 100644 --- a/python/ppt/diff.py +++ b/python/ppt/diff.py @@ -214,7 +214,7 @@ def main(options): return 0 -if __name__ == '__main__': +def get_parser(): parser = argparse.ArgumentParser() parser.add_argument( @@ -264,6 +264,11 @@ def main(options): help='Names of files or directories to compare.', nargs=2) + return parser + + +if __name__ == '__main__': + parser = get_parser() options = parser.parse_args() sys.exit(main(options)) diff --git a/python/ppt/tests/__init__.py b/python/ppt/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/ppt/tests/data/bar.txt b/python/ppt/tests/data/bar.txt new file mode 100644 index 0000000..816f438 --- /dev/null +++ b/python/ppt/tests/data/bar.txt @@ -0,0 +1,4 @@ +abcde +zzzzz +klmnoy +tuvwxy diff --git a/python/ppt/tests/data/file1.txt b/python/ppt/tests/data/file1.txt new file mode 100755 index 0000000..35bf947 --- /dev/null +++ b/python/ppt/tests/data/file1.txt @@ -0,0 +1,24 @@ +Now we are engaged in a great civil war, testing whether that nation, +or any nation so conceived and so dedicated, can long endure. We are +met on a great battle-field of that war. We have come to dedicate a +portion of that field, as a final resting place for those who here +gave their lives that that nation might live. It is altogether +fitting and proper that we should do this. + +But, in a larger sense, we can not dedicate -- we can not consecrate +-- we can not hallow -- this ground. The brave men, living and dead, +who struggled here, have consecrated it, far above our poor power to +add or detract. The world will little note, nor long remember what we +say here, but it can never forget what they did here. It is for us the +living, rather, to be dedicated here to the unfinished work which they +who fought here have thus far so nobly advanced. It is rather for us +to be here dedicated to the great task remaining before us -- that +from these honored dead we take increased devotion to that cause for +which they gave the last full measure of devotion -- that we here +highly resolve that these dead shall not have died in vain -- that +this nation, under God, shall have a new birth of freedom -- and that +government of the people, by the people, for the people, shall not +perish from the earth. + +Abraham Lincoln +November 19, 1863 \ No newline at end of file diff --git a/python/ppt/tests/data/file2.txt b/python/ppt/tests/data/file2.txt new file mode 100755 index 0000000..cfdc12c --- /dev/null +++ b/python/ppt/tests/data/file2.txt @@ -0,0 +1,26 @@ +Now we are engaged in a great civil war, testing whether that nation, +or any nation so conceived and so dedicated, can long endure. We are +met on a great battle-field of that war. We have come to dedicate a +portion of that field, as a final resting place for those who here +gave their lives that that nation might live. It is altogether +fitting and proper that we should do this. + +We also sometimes add a line to a file. + +But, in a larger sense, we can not dedicate -- we can not consecrate +-- we can not hallow -- this ground. The brave men, living and dead, +who struggled here, have consecrated it, far above our poor power to +add or detract. The world will little note, nor long remember what we +say here, but it can never forget what they did here. It is for us the +living, rather, to be dedicated here to the unfinished work which they +who fought here have thus far so nobly advanced. It is rather for us +to be here dedicated to the great task remaining before us -- that +from these honored dead we take increased devotion to that cause for +which they gave the last full measure of devotion -- that we here +highly resolve that these dead shall not have died in vain -- that +this nation, under God, shall have a new birth of freedom -- and that +government of the people, by the people, for the people, shall not +perish from the earth because we changed a line in the file. + +Abraham Lincoln +November 19, 1863 \ No newline at end of file diff --git a/python/ppt/tests/data/foo.txt b/python/ppt/tests/data/foo.txt new file mode 100644 index 0000000..3cc1e54 --- /dev/null +++ b/python/ppt/tests/data/foo.txt @@ -0,0 +1,3 @@ +abcde +fgjij +klmno diff --git a/python/ppt/tests/data/gettysburg_address.txt b/python/ppt/tests/data/gettysburg_address.txt new file mode 100755 index 0000000..23a179c --- /dev/null +++ b/python/ppt/tests/data/gettysburg_address.txt @@ -0,0 +1,25 @@ +Four score and seven years ago our fathers brought forth on this +continent a new nation, conceived in Liberty, and dedicated to the +proposition that all men are created equal. + +Now we are engaged in a great civil war, testing whether that nation +or any nation so conceived and so dedicated, can long endure. We are +met on a great battle-field of that war. We have come to dedicate a +portion of that field, as a final resting place for those who here +gave their lives that that nation might live. It is altogether fitting +and proper that we should do this. + +But, in a larger sense, we can not dedicate--we can not consecrate--we +can not hallow--this ground. The brave men, living and dead, who +struggled here, have consecrated it, far above our poor power to add +or detract. The world will little note, nor long remember what we say +here, but it can never forget what they did here. It is for us the +living, rather, to be dedicated here to the unfinished work which they +who fought here have thus far so nobly advanced. It is rather for us +to be here dedicated to the great task remaining before us—that from +these honored dead we take increased devotion to that cause for which +they gave the last full measure of devotion—that we here highly +resolve that these dead shall not have died in vain—that this nation, +under God, shall have a new birth of freedom—and that government of +the people, by the people, for the people, shall not perish from the +earth. diff --git a/python/ppt/tests/testdiff.py b/python/ppt/tests/testdiff.py new file mode 100644 index 0000000..2a8d935 --- /dev/null +++ b/python/ppt/tests/testdiff.py @@ -0,0 +1,134 @@ +""" testdiff.py - Unit test diff.py. """ + +import ppt.diff +import unittest +import warnings +import os +import io +from contextlib import redirect_stdout, redirect_stderr + +# Locate the data folder relative to this test file. +TEST_DIR = os.path.dirname(__file__) +DATA_DIR = os.path.join(TEST_DIR, 'data') + + +""" mkfile - Utility to populate a file. """ +def mkfile(filename, body=None): + with open(filename, 'w') as f: + f.write(body or filename) + return + + +""" make_example_dir - Utility to create a directory hierarchy of files. """ +def make_example_dir(top): + if not os.path.exists(top): + os.mkdir(top) + curdir = os.getcwd() + os.chdir(top) + + os.mkdir('dir1') + os.mkdir('dir2') + + mkfile('dir1/file_only_in_dir1') + mkfile('dir2/file_only_in_dir2') + + os.mkdir('dir1/dir_only_in_dir1') + os.mkdir('dir2/dir_only_in_dir2') + + os.mkdir('dir1/common_dir') + os.mkdir('dir2/common_dir') + + mkfile('dir1/common_file', 'this file is the same') + mkfile('dir2/common_file', 'this file is the same') + + mkfile('dir1/not_the_same', 'aaaa bbbb cccc\n') + mkfile('dir2/not_the_same', 'dddd eeee ffff\n') + + mkfile('dir1/file_in_dir1', 'This is a file in dir1') + os.mkdir('dir2/file_in_dir1') + + os.chdir(curdir) + + +class TestDiff(unittest.TestCase): + def test_diff_two_files(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + parser = ppt.diff.get_parser() + options = parser.parse_args([ os.path.join(DATA_DIR, 'foo.txt'), os.path.join(DATA_DIR, 'bar.txt') ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.diff.main(options) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """2,3c2,4 +< fgjij +< klmno +--- +> zzzzz +> klmnoy +> tuvwxy +""" + self.assertEqual(captured_stdout, expected_result) + + def test_diff_another_two_files(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + parser = ppt.diff.get_parser() + options = parser.parse_args([ os.path.join(DATA_DIR, 'file1.txt'), os.path.join(DATA_DIR, 'file2.txt') ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.diff.main(options) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """6a7,8 +> +> We also sometimes add a line to a file. +21c23 +< perish from the earth. +--- +> perish from the earth because we changed a line in the file. +""" + self.assertEqual(captured_stdout, expected_result) + + def test_diff_recursively_compare_two_directory_trees(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + # Make the directory hierarchies (dir1 + dir2) + make_example_dir(os.path.join(DATA_DIR, 'example')) + make_example_dir(os.path.join(DATA_DIR, 'example/dir1/common_dir')) + make_example_dir(os.path.join(DATA_DIR, 'example/dir2/common_dir')) + + parser = ppt.diff.get_parser() + options = parser.parse_args([ '-r', os.path.join(DATA_DIR, 'example/dir1'), os.path.join(DATA_DIR, 'example/dir2') ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.diff.main(options) + + # Compare the results. + captured_stdout = out.getvalue() + s = captured_stdout.split('\n') + + self.assertRegex(s[0], r'Only in .+/data/example/dir1:$') + self.assertRegex(s[1], r'dir_only_in_dir1') + self.assertRegex(s[2], r'file_only_in_dir1') + self.assertRegex(s[3], r'Only in .+/data/example/dir2:$') + self.assertRegex(s[4], r'dir_only_in_dir2') + self.assertRegex(s[5], r'file_only_in_dir2') + self.assertRegex(s[6], r'diff .+/data/example/dir1/not_the_same .+/data/example/dir2/not_the_same$') + + diffs = '\n'.join(s[7:11]) + expected_diffs = """1c1 +< aaaa bbbb cccc +--- +> dddd eeee ffff""" + self.assertEqual(diffs, expected_diffs) diff --git a/python/ppt/tests/testwc.py b/python/ppt/tests/testwc.py new file mode 100644 index 0000000..6c05eee --- /dev/null +++ b/python/ppt/tests/testwc.py @@ -0,0 +1,159 @@ +''' testwc.py - Unit test wc.py. ''' + +import ppt.wc +import unittest +import warnings + +import os +import unittest.mock +import io +import sys +from contextlib import redirect_stdout, redirect_stderr + +# Locate the data folder relative to this test file. +TEST_DIR = os.path.dirname(__file__) +DATA_DIR = os.path.join(TEST_DIR, 'data') +DATA_FILE = os.path.join(DATA_DIR, 'gettysburg_address.txt') + +class TestWc(unittest.TestCase): + def test_wc_help(self): + # Capturing the help output is tricky, due to: + # 1. The way -h/--help options work. You have to patch them + # into sys.argv in order to trigger the SystemExit exception. + # 2. You don't get any output to stdout unless you trigger + # the SystemExit exception. + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + parser = ppt.wc.get_parser() + + for h in ['-h', '--help']: + with unittest.mock.patch('sys.argv', ['wc.py', h]), \ + unittest.mock.patch('sys.stdout', new_callable=io.StringIO) as mock_stdout, \ + self.assertRaises(SystemExit) as cm: + parser.parse_args() + + self.assertEqual(cm.exception.code, 0) + self.assertIn('-h, --help show this help message and exit', mock_stdout.getvalue()) + + + def test_wc_help2(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + parser = ppt.wc.get_parser() + + # This works, but can't seem to capture the output via stdout as with the above. + # (Or any other way that I know at this time.) + for h in ['-h', '--help']: + parser.parse_args(h) + + + @unittest.skip('Skipping - mock_stdout is empty.') + def test_wc_help3(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + for h in ['-h', '--help']: + with unittest.mock.patch('sys.stdout', new_callable=io.StringIO) as mock_stdout: + parser = ppt.wc.get_parser() + parser.parse_args(h) + + self.assertIn('-h, --help show this help message and exit', mock_stdout.getvalue()) + + + @unittest.skip('Skipping - io_out and io_err are empty.') + def test_wc_help4(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + for h in ['-h', '--help']: + io_out = io.StringIO() + io_err = io.StringIO() + with redirect_stdout(io_out), redirect_stderr(io_err): + parser = ppt.wc.get_parser() + parser.parse_args(h) + + captured_stdout = io_out.getvalue() + captured_stderr = io_err.getvalue() + + + def test_wc_file(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + parser = ppt.wc.get_parser() + + # Run all options on the file in turn. + # Can't test the help options (-h, --help) because argparse exits after printing the help. + fixture = [ + { 'sw' : '-c', 'val' : 1456 }, + { 'sw' : '--bytes', 'val' : 1456 }, + { 'sw' : '-m', 'val' : 1456 }, + { 'sw' : '--chars', 'val' : 1456 }, + { 'sw' : '-l', 'val' : 25 }, + { 'sw' : '--lines', 'val' : 25 }, + { 'sw' : '-w', 'val' : 302 }, + { 'sw' : '--words', 'val' : 302 }] + + for f in fixture: + # Capture stdout when running. + io_out = io.StringIO() + with redirect_stdout(io_out): + parser = ppt.wc.get_parser() + options = parser.parse_args([f['sw'], DATA_FILE]) + ppt.wc.main(options) + + # Parse the result and compare to expected value. + captured_stdout = io_out.getvalue() + val_int = int(captured_stdout.split()[0]) + self.assertEqual(val_int, f['val']) + + + def test_wc_file2(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + parser = ppt.wc.get_parser() + options = parser.parse_args([DATA_FILE]) + ppt.wc.main(options) + + # Compare the results. + captured_stdout = out.getvalue() + vals_int = [int(v) for v in captured_stdout.split()[0:3]] + expected_vals = [25, 302, 1456] + self.assertEqual(vals_int, expected_vals) + + + def test_wc_stdin(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + parser = ppt.wc.get_parser() + options = parser.parse_args([]) + + # Feed the file in via stdin. + # Unfortunately, passing the file handle doesn't work. + # re.findall() gives this error: + # File "/usr/lib/python3.12/re/__init__.py", line 217, in findall + # return _compile(pattern, flags).findall(string) + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + # TypeError: expected string or bytes-like object, got 'MagicMock' + # + # Therefore, this reads the whole file in (frown) and passes that. + with unittest.mock.patch('sys.stdin') as mock_stdin, open(DATA_FILE, 'r', encoding='utf-8') as data_file: + data_file_contents = data_file.read() + mock_stdin.read.return_value = data_file_contents + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.wc.main(options) + + # Compare the results. + captured_stdout = out.getvalue() + vals_int = [int(v) for v in captured_stdout.split()[0:3]] + expected_vals = [25, 302, 1456] + self.assertEqual(vals_int, expected_vals) + diff --git a/python/ppt/wc.py b/python/ppt/wc.py index 1ecfd91..ff0f371 100644 --- a/python/ppt/wc.py +++ b/python/ppt/wc.py @@ -103,8 +103,8 @@ def main(options): else: wordcountfilenames(options) - -if __name__ == '__main__': + +def get_parser(): parser = argparse.ArgumentParser() parser.add_argument( '-c', @@ -144,5 +144,10 @@ def main(options): 'filenames', help='Names of files whose words are to be counted.', nargs='*') + return parser + + +if __name__ == '__main__': + parser = get_parser() options = parser.parse_args() main(options)