(closes #2936, #1220, #2088) Fix doctests and add CI tests

.github/workflows/python-package.yml

@@ -68,12 +68,11 @@ jobs:
           python-version: '3.14'
       - run: sudo apt-get install -y graphviz doxygen
       - run: python -m pip install --upgrade pip
-      - run: pip install .[doc]
+      - run: pip install -e .[test,doc]
       # Now we can check for warnings and broken links
+      - run: cd doc; make doctest
       - run: cd doc; make html SPHINXOPTS="-W --keep-going"
       - run: cd doc; make linkcheck
-      # TODO #2936: There are many doctest issues, so commenting out for now
-      # - run: cd doc; make doctest
   build:
     if: ${{ github.repository != 'stfc/PSyclone-mirror' }}
     runs-on: ubuntu-latest
@@ -127,12 +126,12 @@ jobs:
         if: ${{ !(matrix.python-version == '3.9') }}
         run: |
           locale
-          pytest -n auto --cov=psyclone --cov-report=xml src/psyclone/tests
+          pytest -n auto --doctest-modules --cov=psyclone --cov-report=xml src/psyclone
       - name: Test with pytest and C Locale
         if: ${{ matrix.python-version == '3.9' }}
         run: |
           locale
-          pytest -n auto --cov=psyclone --cov-report=xml src/psyclone/tests
+          pytest -n auto --doctest-modules --cov=psyclone --cov-report=xml src/psyclone
         env:
           LC_ALL: C
           LANG: C

doc/developer_guide/dependency.rst

@@ -461,7 +461,7 @@ thread-private. Note that this code does not handle the usage of
       # the access information as well as from the symbol table
       # into account.
       access_sequence = var_accesses[signature]
-      if symbol.is_array_access(access_info=access_info):
+      if symbol.is_array_access(access_info=access_sequence):
           # It's not a scalar variable, so it will not be private
           continue
 
@@ -504,7 +504,7 @@ until we find accesses that would prevent parallelisation:
    for next_statement in statements:
        # Add the variable accesses of the next statement to
        # the existing accesses:
-       next_statement.reference_accesses(accesses)
+       accesses = next_statement.reference_accesses()
        # Stop when the next statement can not be parallelised
        # together with the previous accesses:
        if not can_be_parallelised(accesses):
@@ -581,7 +581,7 @@ can be parallelised:
 .. testoutput::
     :hide:
 
-    Error: The write access to 'a(i,i)' and the read access to 'a(i + 1,i + 1)' are dependent and cannot be parallelised. Variable: 'a'.
+    Error: The write access to 'a(i,i)' in 'a(i,i) = j + k' and the read access to 'a(i + 1,i + 1)' in 'a(i,i) = a(i + 1,i + 1)' are dependent and cannot be parallelised. Variable: 'a'.
 
 .. _defusechain:
 

doc/developer_guide/interface_example.py

@@ -35,11 +35,4 @@ def some_function(filename, kernel_path, node=None):
                              it can be raised by different errors.
     :raises GenerationError: same exception, raised by a different error.
 
-    For example:
-
-    >>> from psyclone.generator import generate
-    >>> API="gocean"
-    >>> alg, psy = generate(SOURCE_FILE, api=API)
-    >>> alg, psy = generate(SOURCE_FILE, api=API, kernel_paths=[KERNEL_PATH])
-
     '''

doc/developer_guide/module_manager.rst

@@ -121,7 +121,8 @@ which prints the filenames of all modules used in ``tl_testkern_mod``:
 .. testcode ::
 
     mod_manager = ModuleManager.get()
-    # Add the path to the PSyclone LFRic example codes:
+    # Add the path to the LFRic module directories:
+    mod_manager.add_search_path("../external/lfric_infrastructure/")
     mod_manager.add_search_path("../src/psyclone/tests/test_files/"
                                 "lfric")
 
@@ -134,15 +135,14 @@ which prints the filenames of all modules used in ``tl_testkern_mod``:
         mod_info = mod_manager.get_module_info(module_name)
         print("Module:", module_name, os.path.basename(mod_info.filename))
 
+
 .. testoutput::
 
     Module: argument_mod argument_mod.f90
     Module: constants_mod constants_mod.f90
     Module: fs_continuity_mod fs_continuity_mod.f90
     Module: kernel_mod kernel_mod.f90
 
-
-
 FileInfo
 ========
 

doc/developer_guide/psyir_symbols.rst

@@ -337,7 +337,7 @@ specialisations are possible:
     >>> # The following statement would fail because the initial_value doesn't
     >>> # match the datatype of the symbol:
     >>> # sym2.specialise(DataSymbol, datatype=ScalarType.integer_type(),
-    ...                   initial_value=3.14)
+    >>> #                 initial_value=3.14)
     >>> # The following statement is valid and initial_value is set to 3
     >>> # (and is_constant will default to False):
     >>> sym2.specialise(DataSymbol, datatype=ScalarType.integer_type(), initial_value=3)

doc/developer_guide/transformations.rst

@@ -33,14 +33,6 @@
 .. -----------------------------------------------------------------------------
 .. Written by R. W. Ford, A. R. Porter, S. Siso and N. Nobre, STFC Daresbury Lab
 
-.. testsetup::
-
-    # Define GOCEAN_SOURCE_FILE to point to an existing gocean 1.0 file.
-    GOCEAN_SOURCE_FILE = ("../src/psyclone/tests/test_files/"
-        "gocean1p0/test11_different_iterates_over_one_invoke.f90")
-    # Define NEMO_SOURCE_FILE to point to an existing nemo file.
-    NEMO_SOURCE_FILE = ("../examples/nemo/code/tra_adv.F90")
-
 
 Transformations
 ###############

doc/user_guide/gocean1p0.rst

@@ -196,8 +196,8 @@ Example
 +++++++
 
 PSyclone is distributed with a full example of the use of the
-GOcean Library. See ``<PSYCLONEHOME>/examples/gocean/shallow_alg.f90``.  In what
-follows we will walk through a slightly cut-down example for a
+GOcean Library. See ``<PSYCLONEHOME>/examples/gocean/eg1/shallow_alg.f90``.
+In what follows we will walk through a slightly cut-down example for a
 different program.
 
 The following code illustrates the use of dl_esm_inf for constructing an

doc/user_guide/psyir.rst

@@ -40,7 +40,7 @@
 .. testsetup::
 
     from psyclone.psyir.symbols import DataSymbol, ScalarType, ArrayType
-	from psyclone.psyir.nodes import Reference
+    from psyclone.psyir.nodes import Reference
 
 .. _psyir-ug:
 
@@ -268,7 +268,7 @@ example:
     ...                       ScalarType.Precision.SINGLE)
     >>> bool_type = ScalarType(ScalarType.Intrinsic.BOOLEAN, 4)
     >>> symbol = DataSymbol("rdef", int_type, initial_value=4)
-    >>> scalar_type = ScalarType(ScalarType.Intrinsic.REAL, symbol)
+    >>> scalar_type = ScalarType(ScalarType.Intrinsic.REAL, Reference(symbol))
 
 For convenience ScalarType has static methods to create a number of
 common scalar datatypes:

src/psyclone/alg_gen.py

@@ -72,26 +72,16 @@ class Alg:
     latter allows consistent names to be generated between the
     algorithm (calling) and psy (callee) layers.
 
-    For example:
-
-    >>> from psyclone.algorithm.parse import parse
-    >>> parse_tree, info = parse("argspec.F90")
-    >>> from psyclone.psyGen import PSy
-    >>> psy = PSy(info)
-    >>> from psyclone.alg_gen import Alg
-    >>> alg = Alg(parse_tree, psy)
-    >>> print(alg.gen)
-
-    :param parse_tree: an object containing a parse tree of the \
-        algorithm specification which was produced by the function \
-        :func:`psyclone.parse.algorithm.parse`. Assumes the algorithm \
-        will be parsed by fparser2 and expects a valid program unit, \
+    :param parse_tree: an object containing a parse tree of the
+        algorithm specification which was produced by the function
+        :func:`psyclone.parse.algorithm.parse`. Assumes the algorithm
+        will be parsed by fparser2 and expects a valid program unit,
         program, module, subroutine or function.
     :type parse_tree: :py:class:`fparser.two.utils.Base`
     :param psy: an object containing information about the PSy layer.
     :type psy: :py:class:`psyclone.psyGen.PSy`
-    :param str invoke_name: the name that the algorithm layer uses to \
-        indicate an invoke call. This is an optional argument that \
+    :param str invoke_name: the name that the algorithm layer uses to
+        indicate an invoke call. This is an optional argument that
         defaults to the name "invoke".
 
     '''

src/psyclone/core/symbolic_maths.py

@@ -50,10 +50,13 @@ class SymbolicMaths:
     provides convenience functions for PSyclone. It has a Singleton
     access, e.g.:
 
+    >>> from psyclone.psyir.frontend.fortran import FortranReader
     >>> from psyclone.psyir.backend.fortran import FortranWriter
     >>> from psyclone.core import SymbolicMaths
     >>> sympy = SymbolicMaths.get()
-    >>> # Assume lhs is the PSyIR of 'i+j', and rhs is 'j+i'
+    >>> reader = FortranReader()
+    >>> lhs = reader.psyir_from_expression('i+j')
+    >>> rhs = reader.psyir_from_expression('j+i')
     >>> if sympy.equal(lhs, rhs):
     ...     writer = FortranWriter()
     ...     print(f"'{writer(lhs)}' and '{writer(rhs)}' are equal.")

src/psyclone/domain/gocean/transformations/gocean_const_loop_bounds_trans.py

@@ -70,22 +70,15 @@ class GOConstLoopBoundsTrans(Transformation):
     In practice, the application of the constant loop bounds transformation
     looks something like, e.g.:
 
-    >>> from psyclone.parse.algorithm import parse
-    >>> from psyclone.psyGen import PSyFactory
-    >>> import os
-    >>> TEST_API = "gocean"
-    >>> _, info = parse(os.path.join("tests", "test_files", "gocean1p0",
-    ...                              "single_invoke.f90"),
-    ...                 api=TEST_API)
-    >>> psy = PSyFactory(TEST_API).create(info)
-    >>> invoke = psy.invokes.get('invoke_0_compute_cu')
-    >>> schedule = invoke.schedule
+    >>> from psyclone.tests.utilities import get_psylayer_schedule
+    >>> filename = "single_invoke.f90"
+    >>> schedule = get_psylayer_schedule(filename, api="gocean")
     >>>
-    >>> from psyclone.transformations import GOConstLoopBoundsTrans
+    >>> from psyclone.domain.gocean.transformations import \
+    GOConstLoopBoundsTrans
     >>> clbtrans = GOConstLoopBoundsTrans()
     >>>
     >>> clbtrans.apply(schedule)
-    >>> print(schedule.view())
 
     '''
 

src/psyclone/domain/gocean/transformations/gocean_extract_trans.py

@@ -46,29 +46,24 @@
 
 
 class GOceanExtractTrans(ExtractTrans):
-    ''' GOcean1.0 API application of ExtractTrans transformation \
+    ''' GOcean API application of ExtractTrans transformation
     to extract code into a stand-alone program. For example:
 
-    >>> from psyclone.parse.algorithm import parse
-    >>> from psyclone.psyGen import PSyFactory
-    >>>
-    >>> API = "gocean"
-    >>> FILENAME = "shallow_alg.f90"
-    >>> ast, invokeInfo = parse(FILENAME, api=API)
-    >>> psy = PSyFactory(API, distributed_memory=False).create(invoke_info)
-    >>> schedule = psy.invokes.get('invoke_0').schedule
+    >>> from psyclone.tests.utilities import get_psylayer_schedule
+    >>> filename = "eg1/shallow_alg.f90"
+    >>> schedule = get_psylayer_schedule(filename, "gocean-examples")
     >>>
     >>> from psyclone.domain.gocean.transformations import GOceanExtractTrans
     >>> etrans = GOceanExtractTrans()
     >>>
     >>> # Apply GOceanExtractTrans transformation to selected Nodes
     >>> etrans.apply(schedule.children[0])
-    >>> print(schedule.view())
+
     '''
 
     # ------------------------------------------------------------------------
     def validate(self, node_list, options=None):
-        ''' Perform GOcean1.0 API specific validation checks before applying
+        ''' Perform GOcean API specific validation checks before applying
         the transformation.
 
         :param node_list: the list of Node(s) we are checking.

src/psyclone/domain/gocean/transformations/gocean_loop_fuse_trans.py

@@ -49,17 +49,16 @@ class GOceanLoopFuseTrans(LoopFuseTrans):
     in order to fuse two GOcean loops after performing validity checks (e.g.
     that the loops are over the same grid-point type). For example:
 
-    >>> from psyclone.parse.algorithm import parse
-    >>> from psyclone.psyGen import PSyFactory
-    >>> ast, invokeInfo = parse("shallow_alg.f90")
-    >>> psy = PSyFactory("gocean").create(invokeInfo)
-    >>> schedule = psy.invokes.get('invoke_0').schedule
-    >>> print(schedule.view())
+    >>> from psyclone.tests.utilities import get_psylayer_schedule
+    >>> filename = "eg1/shallow_alg.f90"
+    >>> schedule = get_psylayer_schedule(filename, "gocean-examples")
     >>>
-    >>> from psyclone.transformations import GOceanLoopFuseTrans
+    >>> from psyclone.domain.gocean.transformations import GOceanLoopFuseTrans
     >>> ftrans = GOceanLoopFuseTrans()
-    >>> ftrans.apply(schedule[0], schedule[1])
-    >>> print(schedule.view())
+
+    # Currently produces an error with  "Cannot fuse loops that are over "
+    # "different grid-point types: go_cu and go_cv"
+    # >>> ftrans.apply(schedule[0], schedule[1])
 
     '''
     def __str__(self):

src/psyclone/domain/gocean/transformations/gocean_opencl_trans.py

@@ -64,16 +64,24 @@ class GOOpenCLTrans(Transformation):
     InvokeSchedule. Additionally, it will generate OpenCL kernels for
     each of the kernels referenced by the Invoke. For example:
 
-    >>> from psyclone.parse.algorithm import parse
-    >>> from psyclone.psyGen import PSyFactory
-    >>> API = "gocean"
-    >>> FILENAME = "shallow_alg.f90" # examples/gocean/eg1
-    >>> ast, invoke_info = parse(FILENAME, api=API)
-    >>> psy = PSyFactory(API, distributed_memory=False).create(invoke_info)
-    >>> schedule = psy.invokes.get('invoke_0').schedule
+    >>> from psyclone.tests.utilities import get_psylayer_schedule
+    >>> filename = "eg1/shallow_alg.f90"
+    >>> schedule = get_psylayer_schedule(filename, "gocean-examples")
+    >>>
+    >>> from psyclone.domain.gocean.transformations import (
+    ...     GOMoveIterationBoundariesInsideKernelTrans,
+    ...     GOOpenCLTrans)
+    >>> from psyclone.domain.common.transformations import (
+    ...     KernelModuleInlineTrans)
+    >>> move_trans = GOMoveIterationBoundariesInsideKernelTrans()
+    >>> mod_inline_trans = KernelModuleInlineTrans()
     >>> ocl_trans = GOOpenCLTrans()
-    >>> ocl_trans.apply(schedule)
-    >>> print(schedule.view())
+    >>> for kern in schedule.kernels():
+    ...    # Put kernels in same container and iterate the whole space
+    ...    mod_inline_trans.apply(kern)
+    ...    move_trans.apply(kern)
+    >>> # Commented to prevent generating doctest output .cl files
+    >>> # ocl_trans.apply(schedule)
 
     '''
     # Specify which OpenCL command queue to use for management operations like

src/psyclone/domain/lfric/transformations/lfric_extract_trans.py

@@ -47,23 +47,7 @@
 
 class LFRicExtractTrans(ExtractTrans):
     ''' LFRic API application of ExtractTrans transformation
-    to extract code into a stand-alone program. For example:
-
-    >>> from psyclone.parse.algorithm import parse
-    >>> from psyclone.psyGen import PSyFactory
-    >>>
-    >>> API = "lfric"
-    >>> FILENAME = "solver_alg.x90"
-    >>> ast, invokeInfo = parse(FILENAME, api=API)
-    >>> psy = PSyFactory(API, distributed_memory=False).create(invoke_info)
-    >>> schedule = psy.invokes.get('invoke_0').schedule
-    >>>
-    >>> from psyclone.domain.lfric.transformations import LFRicExtractTrans
-    >>> etrans = LFRicExtractTrans()
-    >>>
-    >>> # Apply LFRicExtractTrans transformation to selected Nodes
-    >>> etrans.apply(schedule.children[0:3])
-    >>> print(schedule.view())
+    to extract code into a stand-alone program.
 
     '''
 

src/psyclone/domain/lfric/transformations/lfric_loop_fuse_trans.py

@@ -48,28 +48,21 @@
 
 @transformation_documentation_wrapper
 class LFRicLoopFuseTrans(LoopFuseTrans):
-    ''' LFRic API specialisation of the
-    :py:class:`base class <LoopFuseTrans>` in order to fuse two LFRic
-    loops after performing validity checks. For example:
+    ''' LFRic API specialisation of the :py:class:`base class <LoopFuseTrans>`
+    in order to fuse two LFRic loops after performing validity checks. For
+    example:
 
-    >>> from psyclone.parse.algorithm import parse
-    >>> from psyclone.psyGen import PSyFactory
-    >>>
-    >>> API = "lfric"
-    >>> FILENAME = "alg.x90"
-    >>> ast, invokeInfo = parse(FILENAME, api=API)
-    >>> psy = PSyFactory(API, distributed_memory=False).create(invoke_info)
-    >>> schedule = psy.invokes.get('invoke_0').schedule
-    >>>
-    >>> from psyclone.domain.lfric.transformations import LFRicLoopFuseTrans
-    >>> ftrans =  LFRicLoopFuseTrans()
-    >>>
-    >>> ftrans.apply(schedule[0], schedule[1])
-    >>> print(schedule.view())
+    .. code-block :: python
+
+        from psyclone.domain.lfric.transformations import LFRicLoopFuseTrans
+        ftrans =  LFRicLoopFuseTrans()
+        ftrans.apply(schedule[0], schedule[1])
 
     The optional argument `same_space` can be set as
 
-    >>> ftrans.apply(schedule[0], schedule[1], {"same_space": True})
+    .. code-block :: python
+
+        ftrans.apply(schedule[0], schedule[1], {"same_space": True})
 
     when applying the transformation.