Contents
GotoBLAS
Download this patch to build version 1.02 on Blue Gene and test with Cobalt:
Patch GotoBLAS 1.02, compile and submit tests using Cobalt:
$ cd GotoBLAS $ patch -p1 < ../GotoBLAS-1.02-bgl.patch $ make $ cd test;make
PETSc
Configuration files for various PETSc versions are located on Frost under the directory /contrib/bgl/petsc.
For instance, here is one way to build petsc-3.0.0-p4 on Frost:
$ cd petsc-3.0.0-p4/ $ cp /contrib/bgl/petsc/petsc-3.0.0-p4/bgl-ibm-goto_lapack.py config/ $ patch -p0 < /contrib/bgl/petsc/petsc-3.0.0-p4/petsc-3.0.0-p4.patch $ ./config/bgl-ibm-goto_lapack.py (set PETSC_ARCH and PETSC_DIR environment variables per configure output) $ make all
Trilinos
example config
CC=mpxlc CXX=mpxlC CXXFLAGS="-DMPICH_IGNORE_CXX_SEEK" F77=mpxlf ./configure \ --prefix=/home/voran/local/trilinos \ --enable-triutils \ --enable-teuchos \ --enable-epetraext \ --enable-epetra \ --enable-aztecoo \ --enable-ifpack \ --enable-amesos \ --enable-mpi \ --with-blas=/contrib/bgl/lib/libblas440.a \ --with-lapack=/contrib/bgl/lib/liblapack440.a \ --with-gnumake
comment these lines out in packages/triutils/src/Triutils_config.h:
/* Define to 1 if you have the <cstdio> header file. */ /* #define HAVE_CSTDIO 1 */ /* #define HAVE_CSTDLIB 1 */
compile
make make install
VASP
The installation instructions for VASP are located here: http://cms.mpi.univie.ac.at/vasp/vasp/Installation_VASP.html
Makefiles that are known to work for VASP 5.2 on Blue Gene/L systems have been placed in the /contrib/bgl/vasp directory on Frost. They are also displayed below for reference. Note that we cannot install VASP system-wide because we are not licensed to do so.
VASP may be built on Frost by following steps like these:
Example VASP build steps
$ cd vasp.5.lib $ cp /contrib/bgl/vasp/makefile.bgl.5.lib Makefile $ make $ cd vasp.5.2 $ cp /contrib/bgl/vasp/makefile.bgl.5.2 Makefile $ make clean $ make
Refer to the VASP documentation for information on configuring experiments: http://cms.mpi.univie.ac.at/vasp/vasp/vasp.html
Makefiles listed below for reference:
/contrib/bgl/vasp/makefile.bgl.5.lib
.SUFFIXES: .inc .f .F #----------------------------------------------------------------------- # Makefile for IBM BG/L #----------------------------------------------------------------------- # C-preprocessor CPP = cpp -P -C $*.F >$*.f FC=mpixlf90 -g CC=mpixlc -g CFLAGS = -O3 -qstrict -qarch=440 FFLAGS = -O3 -qstrict -qarch=440 FREE = -qfree=f90 NOFREE = -qfixed DOBJ = preclib.o timing.o derrf.o dclock.o diolib.o dlexlib.o drdatab.o #----------------------------------------------------------------------- # general rules #----------------------------------------------------------------------- libdmy.a: $(DOBJ) linpack_double.o rm -f libdmy.a ar vq libdmy.a $(DOBJ) linpack_double.o # files which do not require autodouble lapack_double.o: lapack_double.f $(FC) $(FFLAGS) $(NOFREE) -c lapack_double.f lapack_single.o: lapack_single.f $(FC) $(FFLAGS) $(NOFREE) -c lapack_single.f linpack_double.o: linpack_double.f $(FC) $(FFLAGS) $(NOFREE) -c linpack_double.f linpack_single.o: linpack_single.f $(FC) $(FFLAGS) $(NOFREE) -c linpack_single.f .c.o: $(CC) $(CFLAGS) -c $*.c .F.o: $(CPP) $(FC) $(FFLAGS) $(FREE) $(INCS) -c $*.f .F.f: $(CPP) .f.o: $(FC) $(FFLAGS) $(FREE) $(INCS) -c $*.f
/contrib/bgl/vasp/makefile.bgl.5.2
.SUFFIXES: .inc .f .F
#-----------------------------------------------------------------------
# Makefile for RS 6000/ SP2, SP3 and possibly SP4, modified for BG/L
# you might also try the makefile.hlrn supplied by Bernd Kallies <kallies@zib.de>
#
#
# Attention:
# =======================
# present default optimisation is -O3, but you might try -O4 as well
#
# several files are optimized less aggressive (see bottom of the makefile),
# since agressive optimisation of those files caused VASP to bomb
# in one or the other compiler version
# because it was impossible to keep track of which file is not
# properly compiled by which version, all files that
# are problematic in one or the other version are compiled
# with lower optimisation
#
# ZHEEVX was not working properly with some version
# if you have problems remove
# #define USE_ZHEEVX
# from subrot.F, davidson.F and wavepre_noio.F
#
#-----------------------------------------------------------------------
# all CPP processed fortran files have the extension .f
SUFFIX=.f
#-----------------------------------------------------------------------
# fortran compiler and linker
#-----------------------------------------------------------------------
FC=mpixlf90
FCL=$(FC)
#-----------------------------------------------------------------------
# C-preprocessor define any of the flags given below
# MPI generate parallel version
# NGZhalf charge density reduced in Z direction
# wNGZhalf gamma point only reduced in Z direction
# CACHE_SIZE 5001 for SP3 and Power 3
# 32768 for 550,590,3CT
# 8001 595/397 quad word systems
# scaLAPACK use scaLAPACK
#
# IBM
# use_allreduce : force mpi_allreduce usage by block of MPI_BLOCK
#-----------------------------------------------------------------------
# Add -DwNGZhalf for gamma point only
# CPP = /usr/bin/cpp -P -C -DHOST=\"BlueGene\" -DMPI -DNGZhalf \
# -Duse_collective -Davoidalloc -DscaLAPACK \
# -DCACHE_SIZE=4000 -DMPI_BLOCK=50000 $*.F >$*.f
CPP_ = ./preprocess <$*.F | /usr/bin/cpp -P -C -traditional >$*$(SUFFIX)
CPP = $(CPP_) -DHOST=\"BlueGene\" -DMPI -DNGZhalf \
-Duse_collective -Davoidalloc \
-DCACHE_SIZE=2000 -DMPI_BLOCK=50000
#-----------------------------------------------------------------------
# general fortran flags, none required
#-----------------------------------------------------------------------
#FFLAGS = -O3 -qstrict -qarch=440d -qtune=440 -qmaxmem=-1
FFLAGS = -g -qfree=f90 -qstrict
#PK 2009/6/3 No opt Works
#FFLAGS = -O0 -g -qstrict -qarch=450d -qtune=450 -qmaxmem=-1
#-----------------------------------------------------------------------
# optimization:
# optimise for the machine on which the code is compiled
#-----------------------------------------------------------------------
#OFLAG = -O3 -qstrict -qarch=450d -qtune=450 -qmaxmem=-1
OFLAG = -O3 -qarch=440 -qtune=440 -qmaxmem=-1
#PK 2009/6/3 No opt works
#OFLAG = -O0 -g -qstrict -qarch=450d -qtune=450 -qmaxmem=-1
OFLAG_HIGH = $(OFLAG)
OBJ_HIGH = none
OBJ_NOOPT = none
DEBUG = -g
INCS =
INLINE = $(OFLAG)
#-----------------------------------------------------------------------
# maybe one need to set an include path (usually not required)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# options for linking
# the following option increases the size of the data frame
# the default makefile does not include support for scaLAPACK
# since problems with scaLAPACK were reported
#
#-----------------------------------------------------------------------
LIBSCA = /soft/apps/SCALAPACK
LIBBLACS = /soft/apps/BLACS
LIBLOC = /contrib/bgl/lib
# LIBTOOLS = ../lib_bgl
# SCALAPACK = -L$(LIBSCA) -lscalapack -L$(LIBBLACS) -lblacsF77init_MPI-BGP-0 -lblacs_MPI-BGP-0
SCALAPACK =
#ESSL = ../vasp.5.lib/dsygv.o ../vasp.5.lib/dgegv.o -L/opt/ibmmath/essl/4.4/lib -lesslbg
#ESSL = ../vasp.5.lib/dsygv.o ../vasp.5.lib/dgegv.o -L/opt/ibmmath/essl/4.4/lib -lesslbg
#ESSL = -L/soft/apps/ESSL-4.4/lib -lesslbg
BLAS = /contrib/bgl/lib/libgoto.a
#ESSL = ../vasp.4.lib/dsygv.o ../vasp.4.lib/dgegv.o -L/opt/ibmmath/essl/4.3/lib -lesslbg /opt/ibmcmp/xlmass/bg/4.4/bglib/libmass.a /opt/ibmcmp/xlmass/bg/4.4/bglib/libmassv.a
LIB = -L../vasp.5.lib -ldmy $(SCALAPACK) -L$(LIBLOC) -llapack440 $(BLAS)
#-----------------------------------------------------------------------
# specify 3d-fft to be used with VASP
#-----------------------------------------------------------------------
FFT3D = fftmpi.o fftmpi_map.o fft3dfurth.o fft3dlib.o
#-----------------------------------------------------------------------
# general rules and compile lines
#-----------------------------------------------------------------------
BASIC= symmetry.o symlib.o lattlib.o random.o
SOURCE= base.o mpi.o smart_allocate.o xml.o \
constant.o jacobi.o main_mpi.o scala.o \
asa.o lattice.o poscar.o ini.o xclib.o xclib_grad.o \
radial.o pseudo.o mgrid.o gridq.o ebs.o \
mkpoints.o wave.o wave_mpi.o wave_high.o \
$(BASIC) nonl.o nonlr.o nonl_high.o dfast.o choleski2.o \
mix.o hamil.o xcgrad.o xcspin.o potex1.o potex2.o \
metagga.o constrmag.o cl_shift.o relativistic.o LDApU.o \
paw_base.o egrad.o pawsym.o pawfock.o pawlhf.o paw.o \
mkpoints_full.o charge.o dipol.o pot.o \
dos.o elf.o tet.o tetweight.o hamil_rot.o \
steep.o chain.o dyna.o sphpro.o us.o core_rel.o \
aedens.o wavpre.o wavpre_noio.o broyden.o \
dynbr.o rmm-diis.o reader.o writer.o tutor.o xml_writer.o \
brent.o stufak.o fileio.o opergrid.o stepver.o \
chgloc.o fast_aug.o fock.o mkpoints_change.o sym_grad.o \
mymath.o internals.o dimer_heyden.o dvvtrajectory.o vdwforcefield.o \
hamil_high.o nmr.o force.o \
pead.o subrot.o subrot_scf.o pwlhf.o gw_model.o optreal.o davidson.o \
electron.o rot.o electron_all.o shm.o pardens.o paircorrection.o \
optics.o constr_cell_relax.o stm.o finite_diff.o elpol.o \
hamil_lr.o rmm-diis_lr.o subrot_cluster.o subrot_lr.o \
lr_helper.o hamil_lrf.o elinear_response.o ilinear_response.o \
linear_optics.o linear_response.o \
setlocalpp.o wannier.o electron_OEP.o electron_lhf.o twoelectron4o.o \
ratpol.o screened_2e.o wave_cacher.o chi_base.o wpot.o local_field.o \
ump2.o bse.o acfdt.o chi.o sydmat.o
INC=
vasp.bgl: $(SOURCE) $(FFT3D) $(INC) main.o
rm -f vasp.bgl
$(FCL) -o vasp.bgl main.o $(SOURCE) $(FFT3D) $(LIB) $(LINK)
makeparam: $(SOURCE) $(FFT3D) makeparam.o main.F $(INC)
$(FCL) -o makeparam $(LINK) makeparam.o $(SOURCE) $(FFT3D) $(LIB)
zgemmtest: zgemmtest.o base.o random.o $(INC)
$(FCL) -o zgemmtest $(LINK) zgemmtest.o random.o base.o $(LIB)
dgemmtest: dgemmtest.o base.o random.o $(INC)
$(FCL) -o dgemmtest $(LINK) dgemmtest.o random.o base.o $(LIB)
ffttest: base.o smart_allocate.o mpi.o mgrid.o random.o ffttest.o $(FFT3D) $(INC)
$(FCL) -o ffttest $(LINK) ffttest.o mpi.o mgrid.o random.o smart_allocate.o base.o $(FFT3D) $(LIB)
kpoints: $(SOURCE) $(FFT3D) makekpoints.o main.F $(INC)
$(FCL) -o kpoints $(LINK) makekpoints.o $(SOURCE) $(FFT3D) $(LIB)
clean:
-rm -f *.g *.f *.o *.L *.mod ; touch *.F
main.o: main$(SUFFIX)
$(FC) $(FFLAGS) $(DEBUG) $(INCS) -c main$(SUFFIX)
xcgrad.o: xcgrad$(SUFFIX)
$(FC) $(FFLAGS) $(INLINE) $(INCS) -c xcgrad$(SUFFIX)
xcspin.o: xcspin$(SUFFIX)
$(FC) $(FFLAGS) $(INLINE) $(INCS) -c xcspin$(SUFFIX)
makeparam.o: makeparam$(SUFFIX)
$(FC) $(FFLAGS) $(DEBUG) $(INCS) -c makeparam$(SUFFIX)
makeparam$(SUFFIX): makeparam.F main.F
#
# MIND: I do not have a full dependency list for the include
# and MODULES: here are only the minimal basic dependencies
# if one strucuture is changed then touch_dep must be called
# with the corresponding name of the structure
#
base.o: base.inc base.F
mgrid.o: mgrid.inc mgrid.F
constant.o: constant.inc constant.F
lattice.o: lattice.inc lattice.F
setex.o: setexm.inc setex.F
pseudo.o: pseudo.inc pseudo.F
poscar.o: poscar.inc poscar.F
mkpoints.o: mkpoints.inc mkpoints.F
wave.o: wave.inc wave.F
nonl.o: nonl.inc nonl.F
nonlr.o: nonlr.inc nonlr.F
$(OBJ_HIGH):
$(CPP)
$(FC) $(FFLAGS) $(OFLAG_HIGH) $(INCS) -c $*$(SUFFIX)
$(OBJ_NOOPT):
$(CPP)
$(FC) $(FFLAGS) $(INCS) -c $*$(SUFFIX)
fft3dlib_f77.o: fft3dlib_f77.F
$(CPP)
$(F77) $(FFLAGS_F77) -c $*$(SUFFIX)
.F.o:
$(CPP)
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
.F$(SUFFIX):
$(CPP)
$(SUFFIX).o:
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
# special rules
#-----------------------------------------------------------------------
radial.o: radial.F
$(CPP)
$(FC) $(FFLAGS) $(INCS) -O2 -c $*$(SUFFIX)
wave.o: wave.F
$(CPP)
$(FC) $(FFLAGS) $(INCS) -O2 -c $*$(SUFFIX)
metagga.o: metagga.F
$(CPP)
$(FC) $(FFLAGS) $(INCS) -O2 -c $*$(SUFFIX)
nonl.o: nonl.F
$(CPP)
$(FC) $(FFLAGS) $(INCS) -O -c $*$(SUFFIX)
paw.o: paw.F
$(CPP)
$(FC) $(FFLAGS) $(INCS) -O1 -c $*$(SUFFIX)
pseudo.o: pseudo.F
$(CPP)
$(FC) $(FFLAGS) $(INCS) -O1 -c $*$(SUFFIX)