Information on the CAM decomposition, for ESMF folks to verify:
Location of files to describe decomposition:
/fis/cgd/home/erik/CAM_decomps
(NOTE: This archive was moved to MSS under /ERIK/archive/CAM_decomps.c060612.tar.gz on June/12/2006)
Subdirectories. exist for each grid. These directories contain the decomposition descriptions for each case, and softlinks to input files that contain the CAM and CLM input grids.
What the Physics Grid Create would look like:
integer, parameter :: numdims = 2
type(ESMF_LOGICAL) :: periodic(numdims)= (/ESMF_TRUE, ESMF_FALSE/)
character(len=64) :: dimnames(numdims) = (/"longitude","latitude "/)
character(len=64) :: dimunits(numdims) = (/"degrees","degrees"/)
!
! pcols is the max number of columns in a chunk:
! For straight latitude decomposition, pcols=plat
! Otherwise, pcols = 16
! On vector platforms pcols = large value optimized for vectorization loops
!
coord1(1) = 0.0
do lon = 2, plon
coord1(lon) = (londeg(lon,1) + londeg(lon-1,1))*0.5
end do
coord1(plon+1) = 360.0
coord2(1) = -90.0
do lat = 2, plat
coord2(lat) = (latdeg(lat) + latdeg(lat-1))*0.5
end do
coord2(plat+1) = 90.0
MyCount = 0
n = 0
do c = begchunk, endchunk ! Loop over chunks in this processor
MyCount = get_ncols_p(c) + Mycount
do i = 1, get_ncols_p(c) ! Number of columns in a chunk
n = n + 1
myIndices(n,1) = get_lat_p(c,i)
myIndices(n,2) = get_lon_p(c,i)
end do
end do
!
! NOTE: Optionally a DELayout could be created with DE's for each chunk, with chunks
! begchunk to endchunk on virtual DE's that are on the same processor.
! Create and distribute ESMF grid
phys_grid = ESMF_GridCreateHorzLatLon(
coord1, coord2, &
horzstagger=ESMF_GRID_HORZ_STAGGER_A, &
dimnames=dimnames,dimunits=dimunits, &
name="Physics Grid", rc=rc)
call ESMF_GridDistribute(phys_grid, delayout=layout, &
myCount=myCount, &
myIndices=myIndices, rc=rc)
|
What are the different physics chunking options that will be used?
use clmtype, only: clm3, gridcell_type
integer, parameter :: numdims = 2
type(ESMF_LOGICAL) :: periodic(numdims)= (/ESMF_TRUE, ESMF_FALSE/)
character(len=64) :: dimnames(numdims) = (/"longitude","latitude "/)
character(len=64) :: dimunits(numdims) = (/"degrees","degrees"/)
type(gridcell_type), pointer :: gptr ! pointer to gridcell derived subtype
coord1(1) = 0.0
do lon = 2, lsmlon
coord1(lon) = (longxy(lon,1) + longxy(lon-1,1))*0.5
end do
coord1(lsmlon+1) = 360.0
coord2(1) = -90.0
do lat = 2, lsmlat
coord2(lat) = (latixy(lat) + latixy(lat-1))*0.5
end do
coord2(lsmlat+1) = 90.0
call get_proc_bounds( begg, endg, begl, endl, begc, endc, &
begp, endp)
gptr => clm3%g
!
! For CLM NOT all of the grid points will be filled (only roughly 1/3rd)
!
MyCount = endg - begg + 1
n = 0
do g = begg, endg ! Loop over grid points used on this processor, get their global indices
n = n + 1
myIndices(n,1) = gptr%ixy(g)
myIndices(n,2) = gptr%jxy(g)
end do
lnd_grid = ESMF_GridCreateHorzLatLon(
coord1, coord2, &
horzstagger=ESMF_GRID_HORZ_STAGGER_A, &
dimnames=dimnames,dimunits=dimunits, &
name="CLM Land model Grid", rc=rc)
call ESMF_GridDistribute(lnd_grid, delayout=layout, &
myCount=myCount, &
myIndices=myIndices, rc=rc)
|
What about the redistribute between land and atmosphere?
This redistribution goes both directions.
call ESMF_BundleRedistStore(atmBundle, lndBundle, parentVM, a2l_route, &
routeOptions=ESMF_ROUTE_OPTION_SYNC+ESMF_ROUTE_OPTION_PACK_PET, rc)
call ESMF_BundleRedist( atmBundle, lndBundle, a2l_route, rc
|
What are the grids and decompositions that will be used for the performance evaluation?
According to the Evaluation plan there are 2 different grids and a total of 8 different decompositions/grids that will be evaluated. Those are:
Write physics chunked decomposition:
Code that does this is checked in on: cam3_2_38_brnch_wrtdecomps
subroutine cam_writedecomp()
use pmgrid, only: plat, plon
use commap, only: londeg, latdeg
use phys_grid, only: get_ncols_p, get_lat_p, get_lon_p, phys_grid_getopts
use ppgrid, only: pcols, begchunk, endchunk
use spmd_utils, only: iam, npes
use units, only: getunit, freeunit
integer, parameter :: numdims = 2
real(r8) :: coord1(plon+1)
real(r8) :: coord2(plat+1)
integer :: MyCount
integer :: MyIndices(pcols*(endchunk-begchunk+1),numdims)
character(len=256) :: filename
integer :: n, n1, c, j, lat, lon, i ! Indices
integer :: ncols
integer :: unit
integer :: phys_load
!
! pcols is the max number of columns in a chunk:
! For straight latitude decomposition, pcols=plat
! Otherwise, pcols = 16
! On vector platforms pcols = large value optimized for vectorization loops
! Coord1, and coord2 are the coordinate vertices that would be entered on GridCreate
!
coord1(1) = 0.0
do lon = 2, plon
coord1(lon) = (londeg(lon,1) + londeg(lon-1,1))*0.5
end do
coord1(plon+1) = 360.0
coord2(1) = -90.0
do lat = 2, plat
coord2(lat) = (latdeg(lat) + latdeg(lat-1))*0.5
end do
coord2(plat+1) = 90.0
MyCount = 0
n = 0
call phys_grid_getopts( phys_loadbalance_out=phys_load )
unit = getunit( iu=10+iam)
write(filename,1010) plat, plon, iam, phys_load
1010 format( 'CAM_decomp:',i3.3,'x',i3.3, 'task-', i2.2, 'load-',i1.1,'.txt')
write(6,*) 'Write out decomposition on file: ', filename
call shr_sys_flush(6)
open(unit=unit,file=filename,status='REPLACE',form='FORMATTED',action='WRITE')
do c = begchunk, endchunk ! Loop over chunks in this processor
ncols = get_ncols_p(c)
MyCount = ncols + Mycount
end do
write(unit,fmt='(a,i6)') 'Total number of grid points on this PE = ', MyCount
do c = begchunk, endchunk ! Loop over chunks in this processor
ncols = get_ncols_p(c)
n1 = n + 1
write(unit,fmt='(a, 4i6)') 'Chunk id, #columns-in-chunk, start-index, end-index', &
c, ncols, n1, n1+ncols
do i = 1, ncols ! Number of columns in a chunk
n = n + 1
myIndices(n,1) = get_lat_p(c,i)
myIndices(n,2) = get_lon_p(c,i)
end do
write(unit,fmt='(a)') 'Latitude: global indices then coordinate in degrees '
write(unit,fmt='(20i6)') (myIndices(j,1), j = n1,n)
write(unit,fmt='(20f6.1)') (latdeg(myIndices(j,1)), j = n1,n)
write(unit,fmt='(a)') 'Longitude: global indices then coordinate in degrees '
write(unit,fmt='(20i6)') (myIndices(j,2), j = n1,n)
write(unit,fmt='(20f6.1)') (londeg(myIndices(j,2),myIndices(j,1)), j = n1,n)
end do
close(unit)
call freeunit(unit)
write(6,) 'Done writting decomp'
call shr_sys_flush(6)
end subroutine cam_writedecomp
|
Land decomp:
subroutine clm_camWriteDecomp()
use clmtype , only : clm3, gridcell_type
use clm_varpar , only : lsmlon, lsmlat
use clm_varsur , only : latixy, longxy, numlon
use clmtype , only : gridcell_type
use spmd_utils , only : iam, npes
use units , only : getunit, freeunit
use phys_grid , only : phys_grid_getopts
use shr_sys_mod , only : shr_sys_flush
use decompMod , only : get_proc_bounds
integer, parameter :: numdims = 2
type(gridcell_type), pointer :: gptr ! pointer to gridcell derived subtype
real(r8) :: coord1(lsmlon+1,lsmlat+1) ! Longitude
real(r8) :: coord2(lsmlon+1,lsmlat+1) ! Latitude
integer :: lat, lon
integer :: begg, endg, begl, endl, begc, endc, begp, endp
integer, allocatable :: myIndices(:,:)
integer :: MyCount, n, g, j, n1, n2, stride, nn
character(len=256) :: filename
integer :: unit
integer :: phys_load
coord1(:,1) = -90.0
coord2(1,:) = 0.0
do lat = 2, lsmlat
do lon = 2, numlon(lat)
coord1(lon,lat) = (longxy(lon,lat) + longxy(lon,lat-1))*0.5
coord2(lon,lat) = (latixy(lon,lat) + latixy(lon-1,lat))*0.5
end do
coord2(numlon(lat)+1,lat) = 360.0
end do
coord1(:,lsmlat+1) = 90.0
call get_proc_bounds( begg, endg, begl, endl, begc, endc, begp, endp)
gptr => clm3%g
!
! Open file to write out to...
!
call phys_grid_getopts( phys_loadbalance_out=phys_load )
unit = getunit( iu=40+iam)
write(filename,1010) lsmlat, lsmlon, iam, phys_load
1010 format( 'CLM_decomp:',i3.3,'x',i3.3, 'task-', i2.2, 'load-',i1.1,'.txt')
write(6,*) 'Write out decomposition on file: ', filename
call shr_sys_flush(6)
open(unit=unit,file=filename,status='REPLACE',form='FORMATTED',action='WRITE')
!
! For CLM NOT all of the grid points will be filled (only roughly 1/3rd)
!
MyCount = endg - begg + 1
write(unit,fmt='(a,i6)') 'Total number of grid points on this PE = ', MyCount
allocate( myIndices(MyCount,numdims) )
n = 0
do g = begg, endg ! Loop over grid points used on this processor, get their global indices
n = n + 1
myIndices(n,1) = gptr%ixy(g) ! Longitude
myIndices(n,2) = gptr%jxy(g) ! Latitude
end do
!
! Write to file:
!
stride = 14
n2 = 0
do nn = 1, n, stride
n1 = n2+1
n2 = n1 + stride-1
if ( n2 > n ) n2 = n
write(unit,fmt='(a,i6,i6)') 'Write grid points between: n1,n2: ', n1, n2
write(unit,fmt='(a)') 'Longitude: global indices then coordinate in degrees '
write(unit,fmt='(20i7)') (myIndices(j,1), j = n1,n2)
write(unit,fmt='(20f7.1)') (longxy(myIndices(j,1),myIndices(j,2)), j = n1,n2)
write(unit,fmt='(a)') 'Latitude: global indices then coordinate in degrees '
write(unit,fmt='(20i7)') (myIndices(j,2), j = n1,n2)
write(unit,fmt='(20f7.1)') (latixy(myIndices(j,1),myIndices(j,2)), j = n1,n2)
end do
call shr_sys_flush(6)
deallocate( myIndices )
close(unit)
call freeunit(unit)
write(6,*) 'Done writting decomp'
end subroutine clm_camWriteDecomp
|
There is a directory called "cam3_2_46_brnch_mct" that has all of the CAM and CLM code in question. I soft-linked the main program to the top level, so you can see what the top level driver looks like. The CAM decomposition is figured out in the file
"models/atm/cam/src/physics/cam1/phys_grid.F90"
The land model decomposition is figured out in the file
"models/lnd/clm2/src/main/decompMod.F90"
Files that have MCT data-structures in them all have a "MCT_" prefix in them.
So
"models/lnd/clm2/src/main/MCT_lnd_comp.F90"
figures out how to map the CLM internal description of it's decomposition into MCT, for example.
"models/atm/cam/src/control/MCT_atm_comp.F90"
does the same for the atmosphere model.
Each of the previous MCT_ layer subroutines have functions that define a MCT Global Seg Map inside them for that particular component in question. But, they use phys_grid or decompMod methods to figure out how to do this. In your case you could either read the files, or try to use the decomposition methods. The problem with doing the later is that you are likely to need a lot of other code that those two files are dependent on.
Looking for use statements in phys_grid...