Improve support for MPI wrappers (#352)

Documentation/source/site-specific-config.rst

@@ -161,6 +161,64 @@ On the other hand, if no MPI is requested, an MPI-enabled compiler
 might be returned, which does not affect the final result, since
 an MPI compiler just adds include- and library-paths.
 
+
+Compiler Wrapper
+================
+Fab supports the concept of a compiler wrapper, which is typically
+a script that calls the actual compiler. An example for a wrapper is
+`mpif90`, which might call a GNU or Intel based compiler (with additional
+parameter for accessing the MPI specific include and library paths.).
+An example to create a `mpicc` wrapper (note that this wrapper is already
+part of Fab, there is no need to explicitly add this yourself):
+
+.. code-block::
+    :linenos:
+    :caption: Defining an mpicc compiler wrapper
+
+    class Mpicc(CompilerWrapper):
+        def __init__(self, compiler: Compiler):
+            super().__init__(name=f"mpicc-{compiler.name}",
+                             exec_name="mpicc",
+                             compiler=compiler, mpi=True)
+
+The tool system allows several different tools to use the same name
+for the executable, as long as the Fab name is different, i.e. the
+`mpicc-{compiler.name}`. The tool
+repository will automatically add compiler wrapper for `mpicc` and
+`mpif90` for any compiler that is added by Fab. If you want to add
+a new compiler, which can also be invoked using `mpicc`, you need
+to add a compiler wrapper as follows:
+
+.. code-block::
+    :linenos:
+    :caption: Adding a mpicc wrapper to the tool repository
+
+    my_new_compiler = MyNewCompiler()
+    ToolRepository().add_tool(my_new_compiler)
+    my_new_mpicc = Mpicc(MyNewCompiler)
+    ToolRepository().add_tool(my_new_mpicc)
+
+When creating a completely new compiler and compiler wrapper
+as in the example above, it is strongly recommended to add
+the new compiler instance to the tool repository as well. This will
+allow the wrapper and the wrapped compiler to share flags. For example,
+a user script can query the ToolRepository to get the original compiler
+and modify its flags. These modification will then automatically be
+applied to the wrapper as well:
+
+.. code-block::
+    :linenos:
+    :caption: Sharing flags between compiler and compiler wrapper
+
+    tr = ToolRepository()
+    my_compiler = tr.get_tool(Category.C_COMPILER, "my_compiler")
+    my_mpicc = tr.get_tool(Category.C_COMPILER, "mpicc-my_compiler")
+
+    my_compiler.add_flags(["-a", "-b"])
+
+    assert my_mpicc.flags == ["-a", "-b"]
+
+
 TODO
 ====
 At this stage compiler flags are still set in the corresponding Fab
@@ -169,3 +227,8 @@ definition in the compiler objects.
 This will allow a site to define their own set of default flags to
 be used with a certain compiler by replacing or updating a compiler
 instance in the Tool Repository
+
+Also, a lot of content in this chapter is not actually about site-specific
+configuration. This should likely be renamed or split (once we
+have details about the using site-specific configuration, which might be
+once the Baf base script is added to Fab).

source/fab/steps/compile_c.py

@@ -10,7 +10,7 @@
 import logging
 import os
 from dataclasses import dataclass
-from typing import List, Dict, Optional, Tuple
+from typing import cast, Dict, List, Optional, Tuple
 
 from fab import FabException
 from fab.artefacts import (ArtefactsGetter, ArtefactSet, ArtefactStore,
@@ -19,7 +19,7 @@
 from fab.metrics import send_metric
 from fab.parse.c import AnalysedC
 from fab.steps import check_for_errors, run_mp, step
-from fab.tools import Category, CCompiler, Flags
+from fab.tools import Category, Compiler, Flags
 from fab.util import CompiledFile, log_or_dot, Timer, by_type
 
 logger = logging.getLogger(__name__)
@@ -124,9 +124,12 @@ def _compile_file(arg: Tuple[AnalysedC, MpCommonArgs]):
     analysed_file, mp_payload = arg
     config = mp_payload.config
     compiler = config.tool_box[Category.C_COMPILER]
-    if not isinstance(compiler, CCompiler):
-        raise RuntimeError(f"Unexpected tool '{compiler.name}' of type "
-                           f"'{type(compiler)}' instead of CCompiler")
+    if compiler.category != Category.C_COMPILER:
+        raise RuntimeError(f"Unexpected tool '{compiler.name}' of category "
+                           f"'{compiler.category}' instead of CCompiler")
+    # Tool box returns a Tool, in order to make mypy happy, we need
+    # to cast it to be a Compiler.
+    compiler = cast(Compiler, compiler)
     with Timer() as timer:
         flags = Flags(mp_payload.flags.flags_for_path(path=analysed_file.fpath,
                                                       config=config))

source/fab/steps/compile_fortran.py

@@ -14,15 +14,15 @@
 from dataclasses import dataclass
 from itertools import chain
 from pathlib import Path
-from typing import List, Set, Dict, Tuple, Optional, Union
+from typing import cast, Dict, List, Optional, Set, Tuple, Union
 
 from fab.artefacts import (ArtefactsGetter, ArtefactSet, ArtefactStore,
                            FilterBuildTrees)
 from fab.build_config import BuildConfig, FlagsConfig
 from fab.metrics import send_metric
 from fab.parse.fortran import AnalysedFortran
 from fab.steps import check_for_errors, run_mp, step
-from fab.tools import Category, Compiler, Flags, FortranCompiler
+from fab.tools import Category, Compiler, Flags
 from fab.util import (CompiledFile, log_or_dot_finish, log_or_dot, Timer,
                       by_type, file_checksum)
 
@@ -133,9 +133,12 @@ def handle_compiler_args(config: BuildConfig, common_flags=None,
 
     # Command line tools are sometimes specified with flags attached.
     compiler = config.tool_box[Category.FORTRAN_COMPILER]
-    if not isinstance(compiler, FortranCompiler):
-        raise RuntimeError(f"Unexpected tool '{compiler.name}' of type "
-                           f"'{type(compiler)}' instead of FortranCompiler")
+    if compiler.category != Category.FORTRAN_COMPILER:
+        raise RuntimeError(f"Unexpected tool '{compiler.name}' of category "
+                           f"'{compiler.category}' instead of FortranCompiler")
+    # The ToolBox returns a Tool. In order to make mypy happy, we need to
+    # cast this to become a Compiler.
+    compiler = cast(Compiler, compiler)
     logger.info(
         f'Fortran compiler is {compiler} {compiler.get_version_string()}')
 
@@ -263,10 +266,13 @@ def process_file(arg: Tuple[AnalysedFortran, MpCommonArgs]) \
         config = mp_common_args.config
         compiler = config.tool_box.get_tool(Category.FORTRAN_COMPILER,
                                             config.mpi)
-        if not isinstance(compiler, FortranCompiler):
-            raise RuntimeError(f"Unexpected tool '{compiler.name}' of type "
-                               f"'{type(compiler)}' instead of "
+        if compiler.category != Category.FORTRAN_COMPILER:
+            raise RuntimeError(f"Unexpected tool '{compiler.name}' of "
+                               f"category '{compiler.category}' instead of "
                                f"FortranCompiler")
+        # The ToolBox returns a Tool, but we need to tell mypy that
+        # this is a Compiler
+        compiler = cast(Compiler, compiler)
         flags = Flags(mp_common_args.flags.flags_for_path(
             path=analysed_file.fpath, config=config))
 

source/fab/tools/__init__.py

@@ -11,8 +11,8 @@
 from fab.tools.category import Category
 from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc,
                                 Gfortran, GnuVersionHandling, Icc, Ifort,
-                                IntelVersionHandling, MpiGcc, MpiGfortran,
-                                MpiIcc, MpiIfort)
+                                IntelVersionHandling)
+from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90
 from fab.tools.flags import Flags
 from fab.tools.linker import Linker
 from fab.tools.psyclone import Psyclone
@@ -29,6 +29,7 @@
            "CCompiler",
            "Compiler",
            "CompilerSuiteTool",
+           "CompilerWrapper",
            "Cpp",
            "CppFortran",
            "Fcm",
@@ -43,10 +44,8 @@
            "Ifort",
            "IntelVersionHandling",
            "Linker",
-           "MpiGcc",
-           "MpiGfortran",
-           "MpiIcc",
-           "MpiIfort",
+           "Mpif90",
+           "Mpicc",
            "Preprocessor",
            "Psyclone",
            "Rsync",