From 4e48480e4e0cf8277da7183255892207621d06e5 Mon Sep 17 00:00:00 2001
From: Sebastian <35812030+Sebastian-47@users.noreply.github.com>
Date: Thu, 29 Jul 2021 19:53:43 +0200
Subject: [PATCH] adds ScaLAPACK and MUMPS to be integrated in Trilinos and
 PetSc (#246)

* Add ScaLAPACK package

* Add MUMPS package

* Removing the check if ParMETIS is available before installing MUMPS and ScaLAPACK.

* Set the packages depending on whether MKL is available

* fix typo
---
 candi.cfg                                    |  2 +
 deal.II-toolchain/packages/mumps.package     | 85 ++++++++++++++++++
 deal.II-toolchain/packages/petsc.package     | 23 +++--
 deal.II-toolchain/packages/scalapack.package | 93 ++++++++++++++++++++
 deal.II-toolchain/packages/trilinos.package  | 16 ++++
 5 files changed, 214 insertions(+), 5 deletions(-)
 create mode 100644 deal.II-toolchain/packages/mumps.package
 create mode 100644 deal.II-toolchain/packages/scalapack.package

diff --git a/candi.cfg b/candi.cfg
index 3a5dd33..647b57d 100644
--- a/candi.cfg
+++ b/candi.cfg
@@ -61,6 +61,7 @@ PACKAGES="load:dealii-prepare"
 #PACKAGES="${PACKAGES} once:boost"
 #PACKAGES="${PACKAGES} once:numdiff"
 #PACKAGES="${PACKAGES} once:openblas"
+#PACKAGES="${PACKAGES} once:scalapack"
 
 # These packages are not dependencies but are tools used for deal.II development:
 #PACKAGES="${PACKAGES} once:astyle"
@@ -73,6 +74,7 @@ PACKAGES="load:dealii-prepare"
 #PACKAGES="${PACKAGES} once:ginkgo"
 #PACKAGES="${PACKAGES} once:gmsh"
 #PACKAGES="${PACKAGES} once:gsl"
+#PACKAGES="${PACKAGES} once:mumps"
 PACKAGES="${PACKAGES} once:opencascade"
 PACKAGES="${PACKAGES} once:parmetis"
 #PACKAGES="${PACKAGES} once:sundials"
diff --git a/deal.II-toolchain/packages/mumps.package b/deal.II-toolchain/packages/mumps.package
new file mode 100644
index 0000000..dd73efc
--- /dev/null
+++ b/deal.II-toolchain/packages/mumps.package
@@ -0,0 +1,85 @@
+################################################################################
+## MUMPS                                                                      ##
+################################################################################
+
+VERSION=5.4.0.5
+EXTRACTSTO=mumps-${VERSION}
+
+NAME=v${VERSION}
+PACKING=.tar.gz
+
+SOURCE=https://github.com/scivision/mumps/archive/
+CHECKSUM=0cd3bd9ad9860f7c0092cd561931a36e
+
+BUILDCHAIN=cmake
+
+BUILDDIR=${BUILD_PATH}/mumps-${VERSION}
+INSTALL_PATH=${INSTALL_PATH}/mumps-${VERSION}
+
+# Set compilers & compiler options
+if [ ! -z "${CC}" ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_C_COMPILER=${CC}"
+fi
+
+if [ ! -z "${CXX}" ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_CXX_COMPILER=${CXX}"
+fi
+
+if [ ! -z "${FC}" ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_Fortran_COMPILER=${FC}"
+fi
+
+CONFOPTS="${CONFOPTS} \
+      -D CMAKE_C_FLAGS:STRING='-fPIC -g -O3' \
+      -D CMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON"
+
+if [ ${MKL} = ON ]; then
+    # When MKL is provided, MUMPS will search for the required packages by itself.
+    # Therefore we only need to set MKLROOT.
+    CONFOPTS="${CONFOPTS} \
+      -D MKLROOT=${MKL_DIR}"
+else
+  if [ ! -z "${BLAS_DIR}" ]; then
+      CONFOPTS="${CONFOPTS} \
+        -D pc_blas_LIBRARY_DIRS=${BLAS_DIR}/lib "
+  fi
+
+  if [ ! -z "${LAPACK_DIR}" ]; then
+      CONFOPTS="${CONFOPTS} \
+        -D pc_lapack_LIBRARY_DIRS=${LAPACK_DIR}/lib"
+  fi
+
+  if [ ! -z "${SCALAPACK_DIR}" ]; then
+      CONFOPTS="${CONFOPTS} \
+        -D pc_scalapack_LIBRARY_DIRS=${SCALAPACK_DIR}/lib"
+  else
+      # auto build missing packages
+      CONFOPTS="${CONFOPTS} -D autobuild=true"
+  fi
+fi
+
+# Set metis directories
+if [ ! -z "${METIS_DIR}" ]; then
+    CONFOPTS="${CONFOPTS} \
+        -D metis=true \
+        -D METIS_LIBRARY=${METIS_DIR}/lib/libmetis.so \
+        -D METIS_INCLUDE_DIR=${METIS_DIR}/include \
+        -D PARMETIS_LIBRARY=${PARMETIS_DIR}/lib/libparmetis.so "
+fi
+
+
+package_specific_register () {
+    export MUMPS_DIR=${INSTALL_PATH}
+}
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/${EXTRACTSTO}
+    rm -f $CONFIG_FILE
+    echo "
+export MUMPS_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}
diff --git a/deal.II-toolchain/packages/petsc.package b/deal.II-toolchain/packages/petsc.package
index d9dc2d4..f409cce 100644
--- a/deal.II-toolchain/packages/petsc.package
+++ b/deal.II-toolchain/packages/petsc.package
@@ -80,10 +80,23 @@ if [ ! -z "${PARMETIS_DIR}" ]; then
         ${CONFOPTS} \
         --with-parmetis-dir=${PARMETIS_DIR} \
 	--with-metis-dir=${PARMETIS_DIR}"
+fi
+
+if [ ! -z "${SCALAPACK_DIR}" ]; then
+    cecho ${INFO} "PETSc: configuration with SCALAPACK"
+    CONFOPTS="\
+        ${CONFOPTS} \
+        --with-scalapack-dir=${SCALAPACK_DIR}"
+else
+    CONFOPTS="${CONFOPTS} --download-scalapack=1"
+fi
 
-    for external_pkg in scalapack mumps; do
-        CONFOPTS="${CONFOPTS} --download-${external_pkg}=1"
-    done
+if [ ! -z "${MUMPS_DIR}" ]; then
+    cecho ${INFO} "PETSc: configuration with MUMPS"
+    CONFOPTS="${CONFOPTS} \
+        --with-mumps-dir=${MUMPS_DIR}"
+else
+    CONFOPTS="${CONFOPTS} --download-mumps=1"
 fi
 
 #########################################################################
@@ -104,7 +117,7 @@ package_specific_setup () {
 
 package_specific_register () {
     export PETSC_DIR=${INSTALL_PATH}
-    if [ ! -z "${PARMETIS_DIR}" ]; then
+    if [ ! -z "${SCALAPACK_DIR}" ]; then
         export SCALAPACK_DIR=${INSTALL_PATH}
     fi
 }
@@ -116,7 +129,7 @@ package_specific_conf () {
     echo "
 export PETSC_DIR=${INSTALL_PATH}
 " >> $CONFIG_FILE
-    if [ ! -z "${PARMETIS_DIR}" ]; then
+    if [ ! -z "${SCALAPACK_DIR}" ]; then
        echo "export SCALAPACK_DIR=${INSTALL_PATH}" >> $CONFIG_FILE
     fi
 }
diff --git a/deal.II-toolchain/packages/scalapack.package b/deal.II-toolchain/packages/scalapack.package
new file mode 100644
index 0000000..6d6c251
--- /dev/null
+++ b/deal.II-toolchain/packages/scalapack.package
@@ -0,0 +1,93 @@
+################################################################################
+## ScaLAPACK                                                                  ##
+################################################################################
+
+VERSION=2.1.0
+EXTRACTSTO=scalapack-${VERSION}
+
+NAME=v${VERSION}
+PACKING=.tar.gz
+
+SOURCE=https://github.com/Reference-ScaLAPACK/scalapack/archive/
+CHECKSUM=e9ae970238030fa1e8ee022a9d3e6fbb
+
+BUILDCHAIN=cmake
+
+BUILDDIR=${BUILD_PATH}/scalapack-${VERSION}
+INSTALL_PATH=${INSTALL_PATH}/scalapack-${VERSION}
+
+
+
+# Set compilers & compiler options
+if [ ! -z "${CC}" ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_C_COMPILER=${CC}"
+fi
+
+if [ ! -z "${CXX}" ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_CXX_COMPILER=${CXX}"
+fi
+
+if [ ! -z "${FC}" ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_Fortran_COMPILER=${FC}"
+fi
+
+# Set Fortran flags
+if [ -n ${FC} ]; then
+  FORTRAN_MAJOR_VERSION=$(echo __GNUC__ | ${FC} -E -xc - | tail -n 1)
+  if [ ${FORTRAN_MAJOR_VERSION} > 9 ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_Fortran_FLAGS='-g -O3 -fallow-argument-mismatch' "
+  else
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_Fortran_FLAGS='-g -O3' "
+  fi
+fi
+
+
+# Set C flags
+CONFOPTS="${CONFOPTS} \
+  -D CMAKE_C_FLAGS:STRING='-fPIC -g -O3 ' \
+  -D CMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON"
+
+# Set intel MKL options
+if [ "${MKL}" = "ON" ]; then
+    cecho ${INFO} "MUMPS: configuration with MKL"
+
+    CONFOPTS="\
+        ${CONFOPTS} \
+        -D MKLROOT=${MKL_DIR}"
+else 
+  # Set blas and lapack directories (if MKL is not used)
+  if [ ! -z "${BLAS_DIR}" ]; then
+      cecho ${INFO} "SCALAPACK: configuration with BLAS_DIR=${BLAS_DIR}"
+      CONFOPTS="${CONFOPTS} \
+        -D BLAS_LIBRARY_DIRS:STRING=${BLAS_DIR}"
+  fi
+  
+  if [ ! -z "${LAPACK_DIR}" ]; then
+      cecho ${INFO} "SCALAPACK: configuration with LAPACK_DIR=${LAPACK_DIR}"
+      CONFOPTS="${CONFOPTS} \
+        -D LAPACK_LIBRARY_DIRS:STRING=${LAPACK_DIR}"
+  fi
+fi
+
+# BLACS is provided by ScaLAPACK
+package_specific_register () {
+    export SCALAPACK_DIR=${INSTALL_PATH}
+    # only export the BLACS dir when MKL is not provided, so we do not override the MKL BLACS_DIR
+    if [ ${MKL} = OFF ]; then
+      export BLACS_DIR=${INSTALL_PATH}/lib
+    fi
+}
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/${EXTRACTSTO}
+    rm -f $CONFIG_FILE
+    echo "
+export SCALAPACK_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}
diff --git a/deal.II-toolchain/packages/trilinos.package b/deal.II-toolchain/packages/trilinos.package
index 765446e..9ba3c4a 100644
--- a/deal.II-toolchain/packages/trilinos.package
+++ b/deal.II-toolchain/packages/trilinos.package
@@ -90,6 +90,22 @@ if [ ! -z "${LAPACK_DIR}" ]; then
       -D LAPACK_LIBRARY_DIRS:STRING=${LAPACK_DIR}"
 fi
 
+if [ ! -z "${SCALAPACK_DIR}" ]; then
+    cecho ${INFO} "trilinos: configuration with SCALAPACK_DIR=${SCALAPACK_DIR}"
+    CONFOPTS="\
+        ${CONFOPTS} \
+        -D TPL_ENABLE_SCALAPACK:BOOL=ON \
+        -D SCALAPACK_LIBRARY_DIRS=${SCALAPACK_DIR}/lib"
+fi
+
+if [ ! -z "${MUMPS_DIR}" ]; then
+    cecho ${INFO} "trilinos: configuration with MUMPS_DIR=${MUMPS_DIR}"
+    CONFOPTS="${CONFOPTS} \
+      -D TPL_ENABLE_MUMPS=ON \
+      -D MUMPS_INCLUDE_DIRS:STRING=${MUMPS_DIR}/include \
+      -D MUMPS_LIBRARY_DIRS:STRING=${MUMPS_DIR}/lib"
+fi
+
 # Set intel MKL options
 if [ ${MKL} = ON ]; then
     cecho ${INFO} "trilinos: configuration with MKL"
-- 
2.39.5