############################################################################
#
#  Program:  OpenSees
#
#  Purpose:  A Top-level Makefile to create the libraries needed
#            to use the OpenSees framework.
#
#   version created for TACC ranger & their Portland Group Compilers
#
#  Written by: fmk
#  Modified by: Johan Sebastian Yepes Rios
#  Created: 02/2008
#  Modified: 04/2019
#
#
############################################################################

# PUT YOUR HOME DIRECTOREY HERE IF NOT THE DEFAULT ONE IN
HOME  = /home/jyepesr1/source

BASE  =

FE    = $(HOME)/OpenSees/SRC

PROGRAMMING_MODE = PARALLEL_INTERPRETERS

OpenSees_PROGRAM = $(HOME)/OpenSees/bin/OpenSeesMP

# %---------------------------------%
# |  SECTION 2: MAKEFILE CONSTANTS  |
# %---------------------------------%
#
# Specify the constants the are used as control structure variables in the Makefiles.

OPERATING_SYSTEM = LINUX
DEBUG_MODE = NO_DEBUG
RELIABILITY = NO_RELIABILITY

GRAPHICS = NONE

AMDdir       = $(HOME)/OpenSees/OTHER/AMD
BLASdir      = $(HOME)/OpenSees/OTHER/BLAS
CBLASdir     = $(HOME)/OpenSees/OTHER/CBLAS
LAPACKdir    = $(HOME)/OpenSees/OTHER/LAPACK
SUPERLUdir   = $(HOME)/OpenSees/OTHER/SuperLU_5.1.1/SRC
SUPERLU_DISTdir   = $(HOME)/OpenSees/OTHER/SuperLU_DIST_4.3/SRC
ARPACKdir    = $(HOME)/OpenSees/OTHER/ARPACK
UMFPACKdir   = $(HOME)/OpenSees/OTHER/UMFPACK
METISdir     = $(HOME)/OpenSees/OTHER/METIS
CSPARSEdir   = $(HOME)/OpenSees/OTHER/CSPARSE
SRCdir       = $(HOME)/OpenSees/SRC


DIRS        = $(BLASdir) $(CBLASdir) $(LAPACKdir) $(AMDdir) $(CSPARSEdir) \
        $(SUPERLUdir) $(ARPACKdir) $(UMFPACKdir) $(SRCdir) $(METISdir) $(SUPERLU_DISTdir)

# %-------------------------------------------------------%
# | SECTION 4: LIBRARIES                                  |
# |                                                       |
# | The following section defines the libraries that will |
# | be created and/or linked with when the libraries are  |
# | being created or linked with.                         |
# %-------------------------------------------------------%
#
# Note: if vendor supplied BLAS and LAPACK libraries leave the
# libraries blank. You have to get your own copy of the tcl/tk
# library!!
#
# Note: For libraries that will be created (any in DIRS above)
# make sure the directory exsists where you want the library to go!

FE_LIBRARY          = $(HOME)/OpenSees/lib/libOpenSees.a
NDARRAY_LIBRARY     = $(HOME)/OpenSees/lib/libndarray.a  # BJ_UCD jeremic@@ucdavis.edu
MATMOD_LIBRARY      = $(HOME)/OpenSees/lib/libmatmod.a   # BJ_UCD jeremic@@ucdavis.edu
BJMISC_LIBRARY      = $(HOME)/OpenSees/lib/libBJmisc.a  # BJ_UCD jeremic@@ucdavis.edu
LAPACK_LIBRARY 	    = $(HOME)/OpenSees/lib/libLapack.a
BLAS_LIBRARY        = $(HOME)/OpenSees/lib/libBlas.a
SUPERLU_LIBRARY     = $(HOME)/OpenSees/lib/libSuperLU.a
AMD_LIBRARY         = $(HOME)/OpenSees/lib/libAMD.a
CBLAS_LIBRARY       = $(HOME)/OpenSees/lib/libCBlas.a
ARPACK_LIBRARY      = $(HOME)/OpenSees/lib/libArpack.a
UMFPACK_LIBRARY     = $(HOME)/OpenSees/lib/libUmfpack.a
METIS_LIBRARY       = $(HOME)/OpenSees/lib/libMetis.a
CSPARSE_LIBRARY     = $(HOME)/OpenSees/lib/libCSparse.a
DISTRIBUTED_SUPERLU_LIBRARY     = $(HOME)/OpenSees/lib/libDistributedSuperLU.a

TCL_LIBRARY = -ltcl8.5
TCL_INCLUDES =

GRAPHIC_LIBRARY     =

RELIABILITY_LIBRARY =

# WATCH OUT .. These libraries are removed when 'make wipe' is invoked.

WIPE_LIBS	= $(FE_LIBRARY) \
           $(NDARRAY_LIBRARY) \
           $(MATMOD_LIBRARY) \
           $(SUPERLU_LIBRARY) \
           $(ARPACK_LIBRARY) \
           $(AMD_LIBRARY) \
           $(UMFPACK_LIBRARY) \
           $(METIS_LIBRARY) \
           $(LAPACK_LIBRARY) \
           $(BLAS_LIBRARY) \
           $(CBLAS_LIBRARY) \
	   $(DISTRIBUTED_SUPERLU_LIBRARY)

# %---------------------------------------------------------%
# | SECTION 5: COMPILERS                                    |
# |                                                         |
# | The following macros specify compilers, linker/loaders, |
# | the archiver, and their options.  You need to make sure |
# | these are correct for your system.                      |
# %---------------------------------------------------------%

# ###################################################
# # Compilers
# ###################################################


MPI_DIR = $(MPI_HOME)
MPI_BIN = $(MPI_DIR)/bin

CC++            =  mpiicpc
CC              =  mpiicc
FC              =  mpiifort
F90             =  $(FC)
FORTRAN         =  $(FC)

LINKER          =  $(CC++)

AR			= ar
ARCH		= ar

ARFLAGS     = -q
ARCHFLAGS   = -v -q

RANLIB		= ranlib
RANLIBFLAGS =

GRAPHIC_FLAG = -D_NOGRAPHICS

PROGRAMMING_FLAG = -D_PARALLEL_INTERPRETERS

C++FLAGS         = -D_LINUX -D_UNIX  $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(OPT_FLAG) $(COMP_FLAG)\
					$(PROGRAMMING_FLAG)  $(PETSC_FLAG) $(MUMPS_FLAG) \
					-D_TCL85 -D_BLAS -O3 -ffloat-store -qopenmp -std=gnu99

CFLAGS          =  $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG)$(PROGRAMMING_FLAG) $(OPT_FLAG) $(COMP_FLAG) -D_TCL85 -D_BLAS -O2 -qopenmp -std=gnu99
FFLAGS          =  $(OPT_FLAG) $(COMP_FLAG) -O

# Linker
LINKER          = $(CC++)
LINKFLAGS       =

# Misc
MAKE            = /usr/bin/gmake
CD              = cd
ECHO            = echo
RM              = rm
RMFLAGS         = -f
SHELL           = /bin/sh

# %---------------------------------------------------------%
# | SECTION 6: COMPILATION                                  |
# |                                                         |
# | The following macros specify the macros used in         |
# | to compile the source code into object code.            |
# %---------------------------------------------------------%

.SUFFIXES:
.SUFFIXES:	.C .c .f .f90 .cpp .o .cpp

#
# %------------------%
# | Default command. |
# %------------------%
#
.DEFAULT:
	@$(ECHO) "Unknown target $@, try:  make help"
#
# %-------------------------------------------%
# |  Command to build .o files from .f files. |
# %-------------------------------------------%
#

.cpp.o:
	@$(ECHO) Making $@ from $<
	$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@

.C.o:
	@$(ECHO) Making $@ from $<
	$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@
.c.o:
	@$(ECHO) Making $@ from $<
	$(CC) $(CFLAGS) -c $< -o $@
.f.o:
	@$(ECHO) Making $@ from $<
	$(FC) $(FFLAGS) -c $< -o $@

# %---------------------------------------------------------%
# | SECTION 7: OTHER LIBRARIES                              |
# |                                                         |
# | The following macros specify other libraries that must  |
# | be linked with when creating executables. These are     |
# | platform specific and typically order does matter!!     |
# %---------------------------------------------------------%
MACHINE_LINKLIBS  = -L$(BASE)/lib \
                    -L$(HOME)/lib

# PETSC
HAVEPETSC = NO
PETSCINC =
PETSC_LIB =

HAVEMUMPS = YES

ifeq ($(HAVEMUMPS), YES)

MUMPS = YES
MUMPS_FLAG = -D_MUMPS
MUMPS_DIR = /home/jyepesr1/source/mumps
SCOTCHDIR  = /home/jyepesr1/apps/scotch/6.0.6
METISDIR = /home/jyepesr1/apps/metis/5.1.0
PARMETISDIR = /home/jyepesr1/apps/parmetis/4.0.3

PLAT    = LINUX

MUMPS_LIB = -L$(MUMPS_DIR)/lib \
        $(FE)/system_of_eqn/linearSOE/mumps/MumpsSOE.o \
        $(FE)/system_of_eqn/linearSOE/mumps/MumpsSolver.o \
        $(FE)/system_of_eqn/linearSOE/mumps/MumpsParallelSOE.o \
        $(FE)/system_of_eqn/linearSOE/mumps/MumpsParallelSolver.o \
        -ldmumps -lmumps_common -lpord \
        -L$(SCOTCHDIR)/lib/ -lptesmumps -lptscotch -lptscotcherr -lscotch \
		-L$(PARMETISDIR)/lib -lparmetis \
		-L$(METISDIR)/lib -lmetis

MUMPS_INCLUDE = $(MUMPS_DIR)/include

endif

PARALLEL_LIB = -L/home/jyepesr1/apps/scalapack/2.0.2-impi_18.0.2/lib -lscalapack -lreflapack -lrefblas -ltmg

#PARALLEL_LIB = $(MKLROOT)/lib/intel64/libmkl_lapack95_ilp64.a -L$(MKLROOT)/lib/intel64 -lmkl_scalapack_ilp64 \
#				-lmkl_intel_ilp64 -lmkl_sequential -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lpthread -lm -ldl

MACHINE_NUMERICAL_LIBS  = $(ARPACK_LIBRARY) \
			  $(SUPERLU_LIBRARY) \
			  $(UMFPACK_LIBRARY) \
			  $(AMD_LIBRARY) \
			  $(GRAPHIC_LIBRARY)\
			  $(RELIABILITY_LIBRARY) \
			  $(DISTRIBUTED_SUPERLU_LIBRARY) $(CSPARSE_LIBRARY) \
			  $(PETSC_LIB) $(MUMPS_LIB) $(CBLAS_LIBRARY) $(METIS_LIBRARY)

MACHINE_SPECIFIC_LIBS =  $(PARALLEL_LIB) -lifcore -qopenmp

# %---------------------------------------------------------%
# | SECTION 8: INCLUDE FILES                                |
# |                                                         |
# | The following macros specify include files needed for   |
# | compilation.                                            |
# %---------------------------------------------------------%

MACHINE_INCLUDES = -I/usr/include \
	-I$(MKLROOT)/include \
	-I$(HOME)/include   \
	-I$(UMFPACKdir) \
	-I$(SUPERLUdir) \
	-I$(SUPERLU_DISTdir) \
	-I$(MUMPS_INCLUDE)


# this file contains all the OpenSees/SRC includes
include $(FE)/Makefile.incl

TCL_INCLUDES = -I/usr/include/tcl-private/generic

INCLUDES =  $(TCL_INCLUDES) $(FE_INCLUDES) $(MACHINE_INCLUDES)