kokkos: add serial backend explicitly. add kokkos options and kokkos cuda options. (#9276)

var/spack/repos/builtin/packages/kokkos/package.py

@@ -45,13 +45,43 @@ class Kokkos(Package):
     version('2.02.15', 'de41e38f452a50bb03363c519fe20769')
     version('2.02.07', 'd5baeea70109249f7dca763074ffb202')
 
+    variant('serial', default=True, description="enable Serial backend (default)")
     variant('qthreads', default=False, description="enable Qthreads backend")
     variant('cuda', default=False, description="enable Cuda backend")
     variant('openmp', default=False, description="enable OpenMP backend")
 
+    # Kokkos options
+    variant('aggressive_vectorization', default=False,
+            description="set aggressive_vectorization Kokkos option")
+    variant('disable_profiling', default=False,
+            description="set disable_profiling Kokkos option")
+    variant('disable_dualview_modify_check', default=False,
+            description="set disable_dualview_modify_check Kokkos option")
+    variant('enable_profile_load_print', default=False,
+            description="set enable_profile_load_print Kokkos option")
+    variant('compiler_warnings', default=False,
+            description="set compiler_warnings Kokkos option")
+    variant('disable_deprecated_code', default=False,
+            description="set disable_deprecated_code Kokkos option")
+    variant('enable_eti', default=False,
+            description="set enable_eti Kokkos option")
+
+    # CUDA options
+    variant('force_uvm', default=False,
+            description="set force_uvm Kokkos CUDA option")
+    variant('use_ldg', default=False,
+            description="set use_ldg Kokkos CUDA option")
+    variant('rdc', default=False,
+            description="set rdc Kokkos CUDA option")
+    variant('enable_lambda', default=False,
+            description="set enable_lambda Kokkos CUDA option")
+
     gpu_values = ('Kepler30', 'Kepler32', 'Kepler35', 'Kepler37',
                   'Maxwell50', 'Maxwell52', 'Maxwell53',
-                  'Pascal60', 'Pascal61')
+                  'Pascal60', 'Pascal61',
+                  'Volta70', 'Volta72')
+
+    cuda_options = ('force_uvm', 'use_ldg', 'rdc', 'enable_lambda')
 
     # Host architecture variant
     variant(
@@ -71,18 +101,43 @@ class Kokkos(Package):
         description='Set the GPU architecture to use'
     )
 
+    # Checks on Kokkos version and Kokkos options
+    conflicts('+aggressive_vectorization', when='@:2.0.99',)
+    conflicts('+disable_profiling', when='@:2.0.99',)
+    conflicts('+disable_dualview_modify_check', when='@:2.03.04',)
+    conflicts('+enable_profile_load_print', when='@:2.03.04',)
+    conflicts('+compiler_warnings', when='@:2.03.14',)
+    conflicts('+disable_deprecated_code', when='@:2.5.99',)
+    conflicts('+enable_eti', when='@:2.6.99',)
+
     # Check that we haven't specified a gpu architecture
     # without specifying CUDA
     for p in gpu_values:
         conflicts('gpu_arch={0}'.format(p), when='~cuda',
             msg='Must specify CUDA backend to use a GPU architecture.')
 
+    # Check that we haven't specified a Kokkos CUDA option
+    # without specifying CUDA
+    conflicts('+force_uvm', when='~cuda',
+        msg='Must enable CUDA to use force_uvm.')
+    conflicts('+use_ldg', when='~cuda',
+        msg='Must enable CUDA to use use_ldg.')
+    conflicts('+rdc', when='~cuda',
+        msg='Must enable CUDA to use rdc.')
+    conflicts('+enable_lambda', when='~cuda',
+        msg='Must enable CUDA to use enable_lambda.')
+
+    # Check that we haven't asked for a GPU architecture that
+    # the revision of kokkos does not support
+    conflicts('gpu_arch=Volta70', when='@:2.5.99')
+    conflicts('gpu_arch=Volta72', when='@:2.5.99')
+
     # conflicts on kokkos version and cuda enabled
     # see kokkos issue #1296
     # https://github.com/kokkos/kokkos/issues/1296
     conflicts('+cuda', when='@2.5.00:develop',
         msg='Kokkos build system has issue when CUDA enabled'
-        ' in version 2.5.00, 2.7.00, and develop until '
+        ' in version 2.5.00 through 2.7.00, and develop until '
         'issue #1296 is resolved.')
 
     # Specify that v1.x is required as v2.x has API changes
@@ -97,10 +152,14 @@ def install(self, spec, prefix):
             g_args = [
                 '--prefix=%s' % prefix,
                 '--with-hwloc=%s' % spec['hwloc'].prefix,
-                '--with-serial'
             ]
             arch_args = []
+            kokkos_options_args = []
+            cuda_options_args = []
+
             # Backends
+            if 'serial' in spec:
+                g_args.append('--with-serial')
             if '+openmp' in spec:
                 g_args.append('--with-openmp')
             if 'qthreads' in spec:
@@ -115,9 +174,42 @@ def install(self, spec, prefix):
                 arch_args.append(host_arch)
             if gpu_arch:
                 arch_args.append(gpu_arch)
+            # Combined architecture flags
             if arch_args:
                 g_args.append('--arch={0}'.format(','.join(arch_args)))
 
+            # CUDA options
+            if '+force_uvm' in spec:
+                cuda_options_args.append('force_uvm')
+            if '+use_ldg' in spec:
+                cuda_options_args.append('use_ldg')
+            if '+rdc' in spec:
+                cuda_options_args.append('rdc')
+            if '+enable_lambda' in spec:
+                cuda_options_args.append('enable_lambda')
+            if cuda_options_args:
+                g_args.append('--with-cuda-options={0}'
+                              .format(','.join(cuda_options_args)))
+
+            # Kokkos options
+            if '+aggressive_vectorization' in spec:
+                kokkos_options_args.append('aggressive_vectorization')
+            if '+disable_profiling' in spec:
+                kokkos_options_args.append('disable_profiling')
+            if '+disable_dualview_modify_check' in spec:
+                kokkos_options_args.append('disable_dualview_modify_check')
+            if '+enable_profile_load_print' in spec:
+                kokkos_options_args.append('enable_profile_load_print')
+            if '+compiler_warnings' in spec:
+                kokkos_options_args.append('compiler_warnings')
+            if '+disable_deprecated_code' in spec:
+                kokkos_options_args.append('disable_deprecated_code')
+            if '+enable_eti' in spec:
+                kokkos_options_args.append('enable_eti')
+            if kokkos_options_args:
+                g_args.append('--with-options={0}'
+                              .format(','.join(kokkos_options_args)))
+
             generate(*g_args)
             make()
             make('install')