diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index 2d8557e..8e0819f 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/doc/git_reference.md b/doc/git_reference.md index 8effb54..dfed093 100644 --- a/doc/git_reference.md +++ b/doc/git_reference.md @@ -52,6 +52,8 @@ alias gllb='git for-each-ref --format="%(refname:short)" refs/heads' alias glrb='git ls-remote --heads origin |perl -ne '\''print "$1\n" if m|refs/heads(.*)$|'\''' alias gbd="git branch --delete" alias gbdr="git branch -d -r" +alias glog="git log —decorate —oneline —graph" +alias gdiff="git difftool --extcmd=diff --no-prompt" ``` # REPOSITORY diff --git a/python/doc/unittest.md b/python/doc/unittest.md new file mode 100644 index 0000000..e3b2c13 --- /dev/null +++ b/python/doc/unittest.md @@ -0,0 +1,51 @@ +# Python Unit Test + +Example unit test, testmymodule.m, put it in a subdirectory: + +``` +""" testmymodule.py - Unit test mymodule.py. """ + +import mymodule +import unittest + +class TestMymodule(unittest.TestCase): + def test_thing(self): + """ Test doing the thing. """ + result = mymodule.do_the_thing() + expected_result = True + self.assertEqual(result, expected_result) +``` + +Test entire module: + +``` +> python -m unittest mymodule +``` + +Test entire module verbosely (with docstrings): + +``` +> python -m unittest -v mymodule +``` + +Test one specific test case: + +``` +> python -m unittest mymodule.TestMymodule.test_thing +``` + +Discover all unit tests from the current directory on down recursively: + +``` +> PYTHONPATH="$PYTHONPATH:." python -m unittest discover -v -s . +``` + +To skip a unit test, add the `@unittest.skip` decorator: + +``` + @unittest.skip('Skipping - cannot run this unit test for reasons.') + def testsomething(self): + . + . + . +``` \ No newline at end of file diff --git a/python/lib/requirements.txt b/python/lib/requirements.txt index 18c0d8c..5bdb398 100644 --- a/python/lib/requirements.txt +++ b/python/lib/requirements.txt @@ -1 +1 @@ -pyproj==3.4.0 +pyproj>=3.4.0 diff --git a/python/lib/tests/testtime.py b/python/lib/tests/testtime.py index 474e888..2c2e33e 100644 --- a/python/lib/tests/testtime.py +++ b/python/lib/tests/testtime.py @@ -74,8 +74,13 @@ def test_formats(self): self.assertEqual(lib.time.to_military_time('2:00 am'), '02:00') # Leading space(s). - with self.assertRaises(ValueError): - self.assertEqual(lib.time.to_military_time(' 3:00 am'), '03:00') + self.assertEqual(lib.time.to_military_time(' 3:00 am'), '03:00') + + # Trailing space(s). + self.assertEqual(lib.time.to_military_time('3:00 am '), '03:00') + + # Both leading and trailing space(s). + self.assertEqual(lib.time.to_military_time(' 3:00 am '), '03:00') # More than one space separating HH:MM and am/pm. self.assertEqual(lib.time.to_military_time('4:00 pm'), '16:00') diff --git a/python/lib/time.py b/python/lib/time.py index b5d6e18..20e6bb8 100644 --- a/python/lib/time.py +++ b/python/lib/time.py @@ -23,5 +23,11 @@ def date_from_year_doy(year, doy): def to_military_time(x): """ Converts hours/minutes am/pm to military time. """ + + # Normalize whitespace: strip leading/trailing and collapse multiple spaces. + # Need to do this because of differences in behavior of strftime() + # in Python < 3.2 and >= 3.12. + x = ' '.join(x.strip().split()) + return datetime.datetime.strptime(x, "%I:%M %p").strftime("%H:%M") diff --git a/python/ppt/find.py b/python/ppt/find.py index 7387ca1..1af8a34 100644 --- a/python/ppt/find.py +++ b/python/ppt/find.py @@ -1,6 +1,7 @@ -r""" find - Find files recursively and optionally filter with regular expression. +r""" +usage: find.py [-h] [-a] [-e EXCLUDE] [-i] [-d] [-p] path [regex] -usage: find.py [-h] [-a] [-e EXCLUDE] path [regex ...] +Find files (or directories) recursively and optionally filter with regular expression. positional arguments: path Directory path to search. @@ -9,10 +10,11 @@ options: -h, --help show this help message and exit -a, --all Include hidden files and directories. - -e EXCLUDE, --exclude EXCLUDE + -e, --exclude EXCLUDE Exclusion regex to prevent recursing into directories which match. - -i, --ignore-case Prepend "(?i)" to your regex for case-insenstive - matching. + -i, --ignore-case Prepend "(?i)" to your regex for case-insenstive matching. + -d, --dirs Match directory paths only (not files). + -p, --prune When searching paths (with --path), ignore subdirectories of prior matches. """ import argparse @@ -21,44 +23,101 @@ import re -def find(path, aall, regex): +def find_tree( + startpath, + exclude, + aall, + dirsonly, + regex, + prune): + """ Recursively traverse the directory tree starting from + the given path, and apply inclusion/exclusion filtering + along with any regex. + """ + # Exclusion regular expression. excre = None - if args.exclude: - excre = re.compile(args.exclude) - - for root, dirs, files in os.walk(path): - if excre is not None: - # Exclude directories which match excre. - dirs[:] = [d for d in dirs if not excre.search(d)] - - # Skip dot files and directories unless -a/--all is in effect. - if not aall: - # https://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders - # NOTE: os.walk() with topdown = True means dirs and files are modified in-place. - files = [f for f in files if not f[0] == '.'] - dirs[:] = [d for d in dirs if not d[0] == '.'] - - for f in files: - if regex is None: - # No regex specified. Print every file path. - print(os.path.join(root, f)) - else: - # Apply regex. - if re.search(regex, os.path.join(root, f)): - print(os.path.join(root, f)) - - + if exclude: + excre = re.compile(exclude) + + entries = sorted(os.listdir(startpath)) + + # Filter out hidden files/directories unless -a is used. + if not aall: + entries = [e for e in entries if not e.startswith('.')] + + # Separate dirs and files. + dirs = [e for e in entries if os.path.isdir(os.path.join(startpath, e))] + files = [e for e in entries if not os.path.isdir(os.path.join(startpath, e))] + + if excre is not None: + # Exclude directories which match excre. + dirs[:] = [d for d in dirs if not excre.search(d)] + + # Exclude files which match excre. + files[:] = [f for f in files if not excre.search(f)] + + # Combine items to find (dirs first, then files). + all_items = dirs[:] + if not dirsonly: + # Find both directories and files. + all_items += files + + # Use set for fast + reliable directory lookup. + dir_set = set(dirs) + + for i, name in enumerate(all_items): + # Full path down the tree to this item. + fullpath = os.path.join(startpath, name) + + prune_path = False + if regex is None: + # No regex specified, just print. + + # Print out the path if: + # 1. It is a path to a file. + # OR + # 2. We are only searching directory paths and this is a directory path. + if (not (name in dir_set)) or (dirsonly and (name in dir_set)): + print(fullpath) + else: + # Apply regex to full path. + if re.search(regex, fullpath): + print(fullpath) + + # If we matched on a directory, and pruning, + # don't recurse into this directory. + if (name in dir_set) and prune: + prune_path = True + + # Recurse only into subdirectories. + if name in dir_set: + if not prune_path: + find_tree(fullpath, exclude, aall, dirsonly, regex, prune) + + +class PathException(Exception): + pass + + +class ArgumentException(Exception): + pass + + def main(args): if not os.path.isdir(args.path): - print('ERROR: {} is not a directory'.format(args.path)) - else: - if args.ignore_case: - args.regex = '(?i)' + args.regex - find(args.path, args.aall, args.regex) + raise(PathException(f'ERROR - {args.path} is not a directory.')) + if args.path and args.prune and not args.regex: + raise(ArgumentException(f'ERROR - You don\'t need --prune with --path when you don\'t specify a regex.')) + if args.ignore_case: + args.regex = '(?i)' + args.regex + find_tree(args.path, args.exclude, args.aall, args.dirsonly, args.regex, args.prune) + return 0 -if __name__ == '__main__': - parser = argparse.ArgumentParser() + +def get_parser(): + parser = argparse.ArgumentParser( + description='Find files (or directories) recursively and optionally filter with regular expression.') parser.add_argument( '-a', @@ -82,6 +141,22 @@ def main(args): action='store_true', default=False) + parser.add_argument( + '-d', + '--dirs', + dest='dirsonly', + help='Match directory paths only (not files).', + action='store_true', + default=False) + + parser.add_argument( + '-p', + '--prune', + dest='prune', + help='When searching paths (with --path), ignore subdirectories of prior matches.', + action='store_true', + default=False) + parser.add_argument( 'path', help='Directory path to search.') @@ -91,7 +166,10 @@ def main(args): help='Optional regular expression to match on.', nargs='?', default=None) + return parser +if __name__ == '__main__': + parser = get_parser() args = parser.parse_args() sys.exit(main(args)) diff --git a/python/ppt/tests/testfind.py b/python/ppt/tests/testfind.py new file mode 100644 index 0000000..bae925d --- /dev/null +++ b/python/ppt/tests/testfind.py @@ -0,0 +1,161 @@ +""" testfind.py - Unit test diff.py. """ + +import ppt.find +import unittest +import warnings +import os +import io +from contextlib import redirect_stdout, redirect_stderr + +# Locate the data folder relative to this test file. +DATA_DIR = '.' + + +""" mkfile - Utility to populate a file. """ +def mkfile(filename, body=None): + with open(filename, 'w') as f: + f.write(body or filename) + + +""" make_example_dir - Utility to create a directory hierarchy of files. """ +def make_example_dir(top): + if not os.path.exists(top): + os.mkdir(top) + curdir = os.getcwd() + os.chdir(top) + + os.makedirs('xxx/sub1/yyy', exist_ok=True) + mkfile('xxx/sub1/yyy/aaa.txt') + mkfile('xxx/sub1/yyy/aaa_bbb.txt') + + os.makedirs('xxx/sub2/xxx', exist_ok=True) + mkfile('xxx/sub2/xxx/ccc.txt') + mkfile('xxx/sub2/xxx/ccc_ddd.txt') + + os.chdir(curdir) + + +class TestFind(unittest.TestCase): + def test_find_files(self): + """ Find all the files under the specified directory. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + find_test_tree = os.path.join(DATA_DIR, 'find_test_tree') + + make_example_dir(find_test_tree) + + parser = ppt.find.get_parser() + args = parser.parse_args([ find_test_tree ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.find.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./find_test_tree/xxx/sub1/yyy/aaa.txt +./find_test_tree/xxx/sub1/yyy/aaa_bbb.txt +./find_test_tree/xxx/sub2/xxx/ccc.txt +./find_test_tree/xxx/sub2/xxx/ccc_ddd.txt +""" + self.assertEqual(captured_stdout, expected_result) + + def test_find_files_regex(self): + """ Find all the files whose name matches the regex. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + find_test_tree = os.path.join(DATA_DIR, 'find_test_tree') + + make_example_dir(find_test_tree) + + parser = ppt.find.get_parser() + args = parser.parse_args([ find_test_tree, 'aaa' ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.find.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./find_test_tree/xxx/sub1/yyy/aaa.txt +./find_test_tree/xxx/sub1/yyy/aaa_bbb.txt +""" + self.assertEqual(captured_stdout, expected_result) + + def test_find_path(self): + """ Find all directories. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + find_test_tree = os.path.join(DATA_DIR, 'find_test_tree') + + make_example_dir(find_test_tree) + + parser = ppt.find.get_parser() + args = parser.parse_args([ '--dirs', find_test_tree ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.find.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./find_test_tree/xxx +./find_test_tree/xxx/sub1 +./find_test_tree/xxx/sub1/yyy +./find_test_tree/xxx/sub2 +./find_test_tree/xxx/sub2/xxx +""" + self.assertEqual(captured_stdout, expected_result) + + def test_find_path_regex(self): + """ Find all directories that match the regex. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + find_test_tree = os.path.join(DATA_DIR, 'find_test_tree') + + make_example_dir(find_test_tree) + + parser = ppt.find.get_parser() + args = parser.parse_args([ '--dirs', find_test_tree, 'sub2' ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.find.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./find_test_tree/xxx/sub2 +./find_test_tree/xxx/sub2/xxx +""" + self.assertEqual(captured_stdout, expected_result) + + def test_find_path_prune(self): + """ Find all directories that match regex but prune subdirectories. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + find_test_tree = os.path.join(DATA_DIR, 'find_test_tree') + + make_example_dir(find_test_tree) + + parser = ppt.find.get_parser() + args = parser.parse_args([ '--dirs', '--prune', find_test_tree, 'xxx']) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.find.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./find_test_tree/xxx +""" + self.assertEqual(captured_stdout, expected_result) diff --git a/python/ppt/tests/testtree.py b/python/ppt/tests/testtree.py new file mode 100644 index 0000000..59fb2a9 --- /dev/null +++ b/python/ppt/tests/testtree.py @@ -0,0 +1,178 @@ +""" testtree.py - Unit test tree.py. """ + +import ppt.tree +import unittest +import warnings +import os +import io +from contextlib import redirect_stdout, redirect_stderr + +# Locate the data folder relative to this test file. +DATA_DIR = '.' + + +""" mkfile - Utility to populate a file. """ +def mkfile(filename, body=None): + with open(filename, 'w') as f: + f.write(body or filename) + return + + +""" make_example_dir - Utility to create a directory hierarchy of files. """ +def make_example_dir(top): + if not os.path.exists(top): + os.mkdir(top) + curdir = os.getcwd() + os.chdir(top) + + os.makedirs('xxx/sub1/yyy', exist_ok=True) + mkfile('xxx/sub1/yyy/aaa.txt') + mkfile('xxx/sub1/yyy/aaa_bbb.txt') + mkfile('xxx/sub1/yyy/__thing__') + mkfile('xxx/sub1/yyy/AAAAAA.txt') + + os.makedirs('xxx/sub2/xxx', exist_ok=True) + mkfile('xxx/sub2/xxx/ccc.txt') + mkfile('xxx/sub2/xxx/ccc_ddd.txt') + mkfile('xxx/sub2/xxx/.dotfile') + os.makedirs('xxx/sub2/.dotdir', exist_ok=True) + mkfile('xxx/sub2/.dotdir/somefile.txt') + + os.chdir(curdir) + + +class TestTree(unittest.TestCase): + def test_full_tree(self): + """ Test full tree with no options. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + tree_test_tree = os.path.join(DATA_DIR, 'tree_test_tree') + + make_example_dir(tree_test_tree) + + parser = ppt.tree.get_parser() + args = parser.parse_args([ tree_test_tree ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.tree.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./tree_test_tree +└── xxx/ + ├── sub1/ + │ └── yyy/ + │ ├── __thing__ + │ ├── aaa.txt + │ ├── aaa_bbb.txt + │ └── AAAAAA.txt + └── sub2/ + └── xxx/ + ├── ccc.txt + └── ccc_ddd.txt +""" + self.assertEqual(captured_stdout, expected_result) + + def test_full_tree_all(self): + """ Test full tree with -a 'all' option. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + tree_test_tree = os.path.join(DATA_DIR, 'tree_test_tree') + + make_example_dir(tree_test_tree) + + parser = ppt.tree.get_parser() + args = parser.parse_args([ '-a', tree_test_tree ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.tree.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./tree_test_tree +└── xxx/ + ├── sub1/ + │ └── yyy/ + │ ├── __thing__ + │ ├── aaa.txt + │ ├── aaa_bbb.txt + │ └── AAAAAA.txt + └── sub2/ + ├── .dotdir/ + │ └── somefile.txt + └── xxx/ + ├── .dotfile + ├── ccc.txt + └── ccc_ddd.txt +""" + self.assertEqual(captured_stdout, expected_result) + + @unittest.skip('Skipping - unsorted order is indeterminate across operating systems.') + def test_full_tree_unsorted(self): + """ Test full tree with -U unsorted option. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + tree_test_tree = os.path.join(DATA_DIR, 'tree_test_tree') + + make_example_dir(tree_test_tree) + + parser = ppt.tree.get_parser() + args = parser.parse_args([ '-U', tree_test_tree ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.tree.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./tree_test_tree +└── xxx/ + ├── sub1/ + │ └── yyy/ + │ ├── aaa.txt + │ ├── AAAAAA.txt + │ ├── __thing__ + │ └── aaa_bbb.txt + └── sub2/ + └── xxx/ + ├── ccc_ddd.txt + └── ccc.txt +""" + self.assertEqual(captured_stdout, expected_result) + + def test_dir_only(self): + """ Test full tree -d directories only option. """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + + tree_test_tree = os.path.join(DATA_DIR, 'tree_test_tree') + + make_example_dir(tree_test_tree) + + parser = ppt.tree.get_parser() + args = parser.parse_args([ '-d', tree_test_tree ]) + + # Capture stdout when running. + out = io.StringIO() + with redirect_stdout(out): + ppt.tree.main(args) + + # Compare the results. + captured_stdout = out.getvalue() + expected_result = """./tree_test_tree +└── xxx/ + ├── sub1/ + │ └── yyy/ + └── sub2/ + └── xxx/ +""" + self.assertEqual(captured_stdout, expected_result) + diff --git a/python/ppt/tree.py b/python/ppt/tree.py new file mode 100644 index 0000000..7b0d0d1 --- /dev/null +++ b/python/ppt/tree.py @@ -0,0 +1,137 @@ +r""" +usage: tree.py [-h] [-d] [-U] [-a] [path] + +Python clone of the 'tree' command. + +positional arguments: + path Directory to start from (default: current directory). + +options: + -h, --help show this help message and exit + -d List directories only. + -U Do not sort. Use raw filesystem order (like tree -U). + -a Include hidden dot files/directories. +""" +import os +import argparse +import sys + + +def sort_key(name: str) -> str: + """ Push hidden items (starting with . or _) to the front when sorting. """ + if name.startswith(('.', '_')): + return ' ' + name.lower() + return name.lower() + + +def print_tree( + startpath: str, + prefix: str = '', + show_files: bool = True, + sort_enabled: bool = True, + show_all: bool = False): + """ Recursive tree printer - mimics the tree command. """ + try: + entries = os.listdir(startpath) + + # Filter out hidden files/directories unless -a is used. + if not show_all: + entries = [e for e in entries if not e.startswith('.')] + + # Separate dirs and files. + dirs = [e for e in entries if os.path.isdir(os.path.join(startpath, e))] + files = [e for e in entries if not os.path.isdir(os.path.join(startpath, e))] + + # Apply sorting if requested. + if sort_enabled: + dirs = sorted(dirs, key=sort_key) + files = sorted(files, key=sort_key) + + # Combine items to print (dirs first, then files). + all_items = dirs[:] + if show_files: + all_items += files + + # Use set for fast + reliable directory lookup. + dir_set = set(dirs) + + for i, name in enumerate(all_items): + is_last = (i == len(all_items) - 1) + connector = '└── ' if is_last else '├── ' + + # Only directories get trailing '/'. + display_name = name + '/' if name in dir_set else name + + print(prefix + connector + display_name) + + # Recurse only into directories. + if name in dir_set: + new_prefix = prefix + (' ' if is_last else '│ ') + print_tree( + os.path.join(startpath, name), + new_prefix, + show_files=show_files, + sort_enabled=sort_enabled, + show_all=show_all) + + except OSError as e: + # Option 2: Show a helpful message on error but continue traversing + error_msg = str(e).strip() + print(prefix + '└── [error: ' + error_msg + ']', file=sys.stderr) + + +class PathException(Exception): + pass + + +def main(args): + result = 0 + try: + if not os.path.isdir(args.path): + raise PathException(f'tree.py ERROR: {args.path} is not a directory.') + + print(args.path) + print_tree( + startpath=args.path, + show_files=not args.d, + sort_enabled=not args.U, + show_all=args.a) + except PathException as e: + print(e, file=sys.stderr) + result = 1 + except KeyboardInterrupt: + print('\nInterrupted.', file=sys.stderr) + result = 130 + except Exception as e: # fallback for unexpected errors + print(f'Unexpected error: {e}', file=sys.stderr) + result = 1 + + return result + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Python clone of the \'tree\' command.') + parser.add_argument( + 'path', nargs='?', + default='.', + help='Directory to start from (default: current directory).') + parser.add_argument( + '-d', + action='store_true', + help='List directories only.') + parser.add_argument( + '-U', + action='store_true', + help='Do not sort. Use raw filesystem order (like tree -U).') + parser.add_argument( + '-a', + action='store_true', + help='Include hidden dot files/directories.') + return parser + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + sys.exit(main(args))