Parallel UDF on Linux OS

1. Parallel UDF on Linux OS

1 Parallel UDF on Linux OS

Platform: centOS

Goal: imposing user defined 2nd stokes wave on velocity inlet BC

1.1 Steps

Edit source code
Setup the Directory structure
Build the UDF library
Load the UDF Library Load and Hood the UDF to specified BCs in journal file

1.1.1 Setup the Directory structure

In your working directory, make a directory that will store your UDF library (for example, libudf).

$ mkdir libudf

directory structure:

libudf
  + lnamd64
    - 3ddp_host
      + udf.c
      + makefile
    - 3ddp_node
      + udf.c
      + makefile
    - user.udf
  + src
    - user.udf
    - udf.c
    - makefile
  + makefile

Copy makefile.udf2 to the library directory and and name it Makefile

$ cp /share/apps/ansys/17.0.0/v170/fluent/fluent17.0.0/src/udf/makefile.udf2 /data/home/exw692/UDF/libudf/

Notes:

Installation path of makefile on Apocrita

/share/apps/ansys/17.0.0/v170/fluent/fluent17.0.0/src/udf/makefile.udf /share/apps/ansys/17.0.0/v170/Icepak/icepak17.0/Fluent.Inc/fluent17.0.0/src/udf/makefile.udf

/share/apps/ansys/18.0.0/v180/AFD/afd/linx64/runTimeLibraries/fluent/fluent18.0.0/src/udf/makefile.udf /share/apps/ansys/18.0.0/v180/fluent/fluent18.0.0/src/udf/makefile.udf /share/apps/ansys/18.0.0/v180/Icepak/icepak18.0/Fluent.Inc/fluent18.0.0/src/udf/makefile.udf

make a source directory named “src”, In the “libudf” library directory
Copy source file (udf.c) to /src directory
Copy makefile.udf to the /src directory

cp /share/apps/ansys/17.0.0/v170/fluent/fluent17.0.0/src/udf/makefile.udf /data/home/exw692/UDF/libudf/src

cp /share/apps/ansys/18.0.0/v180/fluent/fluent18.0.0/src/udf/makefile.udf /data/home/exw692/UDF/libudf/src

create two sub-directories in directory named “lnamd64”

$ mkdir 3ddp_node
$ mkdir 3ddp_host

Possible file names

2d or 3d single-precision serial 2D or 3D

2ddp or 3ddp double-precision serial 2D or 3D

2d_node and 2d_host single-precision parallel 2D

3d_node and 3d_host single-precision parallel 3D

2ddp_node and 2ddp_host double-precision parallel 2D

3ddp_node and 3ddp_host double-precision parallel 3D

Copy user.udf from path /ansys_inc/v171/fluent/fluent17.1.0/src/user.udf to all the sub-folders, libudf/lnamd64/3ddp_host, libudf/lnamd64/3ddp_node).
copy the “udf.h” Header File in Your Files, /libudf, /src

$ cp /share/apps/ansys/17.0.0/v170/fluent/fluent17.0.0/src/udf/udf.h /data/home/exw692/UDF/libudf/src

1.1.2 Build the UDF Library

After you have set up the folder structure and put the files in the proper places, you can compile and build the shared library using the TUI.

Edit every user.udf file

set the following parameters: CSOURCES, HSOURCES, and ANSYS Fluent path.

CSOURCES = vertical_component_2nd_stokes_wave.c x_component_2nd_stokes_wave.c

HSOURCES = udf.h

FLUENT_INC=/share/apps/ansys/17.0.0/v170/fluent

#CSOURCES: The name of your source file(s)

#HSOURCES: name of head file

#FLUENT_INC: fluent installation path

execute the Makefile

$ make "FLUENT_ARCH=lnamd64"

In your library directory (for example, libudf), execute the Makefile by typing a command that begins

with make and includes the architecture of the machine you will run ANSYS Fluent on, which you identified

in a previous step.

1.1.3 Load the UDF Library and hook UDF to specified BCs using a script (job.jou)

upload the script: job.jou

 rc e387_UDF.cas.gz

rd e387_UDF_800.dat.gz

;load UDF library "libudf"

/define/user-defined/compiled-functions/ load "libudf"

;set boundary condition at velocity inlet

/define boundary-conditions velocity-inlet inlet no yes yes no 0 yes no 0 yes yes "udf" "x_velocity::libudf" yes yes "udf"  "vertical_velocity::libudf" yes no 1 no 1


;set drag moment and monitor in y coordinate on "blades" surface

/solve/monitors/force/unscaled? Yes

/solve/monitors/force/set-drag-monitor cd yes blades () no yes cd-1 no no 0 1 0

/solve/monitors/force/set-moment-monitor moment yes blades () no yes cm-history no no 0 0 0 0 1 0


;save residuals

(display "Save the residual in a file") (newline)

    (let ((writefile (lambda (p)

    (define np (length (residual-history "iteration")))

    (let loop ((i 0))

    (if (not (= i np))

    (begin (define j (+ i 1))

    (display (list-ref (residual-history "iteration") (- np j)) p) (display " " p)

    (display (list-ref (residual-history "continuity") (- np j)) p) (display " " p)

    (display (list-ref (residual-history "x-velocity") (- np j)) p) (display " " p)

    (display (list-ref (residual-history "y-velocity") (- np j)) p) (display " " p)

    (display (list-ref (residual-history "z-velocity") (- np j)) p) (display " " p)

    (display (list-ref (residual-history "k") (- np j)) p) (display " " p)

    (display (list-ref (residual-history "omega") (- np j)) p)

    (newline p)

    (loop (+ i 1))

    )

    )

    )

    ) )

    (output-port (open-output-file "residual_1000_e387_udf.dat")))

    (writefile output-port)

    (close-output-port output-port))



solve/set/time-step 0.02

solve/dual-time-iterate 100 20

wd e387_UDF_900.dat.gz

exit

yes

1.1.4 submitting your job on HPC

upload the scripts: job.jou, job.sh

To submit a Parallel job (multiple nodes)

$ qsub job.sh

Here is an example of a Fluent 3d douple precision job running on 64 cores across 2 nxv nodes.

#For parallel jobs, each node is booked for exclusive use and uses all available memory on each node: 
you do not need to specify a memory  requirement.
# add "-l infiniband=" command
#!/bin/bash
#$ -cwd                       # Set the working directory for the job       
#$ -j y                          #combine the standard output and the standard error stream
#$ -m bea                #get notifications on job start, finish and abortion.
#$ -pe parallel 64      #parallel environment, multiple nodes, 64 cores (2 nxv nodes: 64/32=2)
#$ -l h_rt=4:0:0        #running time, 4 hour
#$ -l infiniband=nxv  #node type: nxv node, each node has 32 cores
#$ -l ansys=64          #request ansys licence for 64 cores

module load ansys
fluent 3ddp -g -rsh -t$NSLOTS -pinfiniband -i udf.jou

-pinfiniband #-p[interconnect], specifies the type of interconnect

1.2 udf.c – UDF code for 2nd stokes wave

/*second order stokes wave at inlet Boundary, wave velocity components are from equations 3.27 and 3.58, Pengzhi lin. numerical modeling of water waves. CRC press, 2008*/
#include "udf.h"
#define pi 3.14159265359 /*define  constants*/
#define U  0.6 /*free stream velocity*/
#define H 0.076 /*wave height*/
#define g 9.81 /*gravity acceleration*/
#define L 4.8 /*wave length*/
#define d 1.6 /*water depth*/
#define T 1.456/*effective wave period (include doppler effect)*/

/*DEFINE_PROFILE: define an inlet velocity profile  that varies as a function of z coordinates or t.*/
DEFINE_PROFILE(x_velocity,ft,var) /* DEFINE Macros, ft is a thread; var:index */
{
    /*define variables*/
    real r[ND_ND]; /*Coordinates, r[0] mean x coordinates, r[1] means y coordinates*/
    real k; /*wave number*/
    real z; /*z(vertical) axis, gravity direction*/
    real omega; /*effective wave angular velocity*/
    real t; /*t*/
    k = 2.0*pi/L;     /*assign values to variables*/
    omega=2.0*pi/T;
    t = CURRENT_TIME; /* Special Fluent macro, current running t */

    face_t f; /* "f" is a face index for each face on the boundary */
   
    begin_f_loop(f,ft)/* face loop macro ,loop over all faces in a given face thread,i.e. "ft" */
    {      
         F_CENTROID(r,f,ft); /*F_CENTROID finds the coordinate position of the centroid of the face "f" and stores the coordinates in the "r" array */
        z =r[1]; /* r[1] is y coordinate,r[2] is z coordinate */
      
F_PROFILE(f,ft,var) = U + H*g*k*cosh(k*(z-0.782+d))*cos(-omega*t)/(2.0*(omega-k*U)*cosh(k*d)) + 3.0*H*H*(omega-k*U)*k*cosh(2.0*k*(z-0.782+d))*cos(-2.0*omega*t)/(16.0*pow(sinh(k*d),4.0));
/*x-velocity component (flow direction): u_r+U= U + H*g*k*cosh(k*(z+d))*cos(-omega*t)/(2.0*(omega-k*U)*cosh(k*d)) + 3.0*H*H*(omega-k*U)*k*cosh(2.0*k*(z+d))*cos(-2.0*omega*t)/(16.0*pow(sinh(k*d),4.0)); z=0 is the mean free surface levelin the theory model, however, free surface level is z=-0.782m in the Fluent geometry model*/

    }
    end_f_loop(f,ft)
}
vertical component of 2nd stokes wave
/*vertical-velocity component (flow direction) of second order stokes wave at inlet Boundary, wave velocity components are from equations 3.27 and 3.58, Pengzhi lin. numerical modeling of water waves. CRC press, 2008*/
#include "udf.h"
#define pi 3.14159265359 /*define  constants*/
#define U  0.6 /*free stream velocity*/
#define H 0.076 /*wave height*/
#define g 9.81 /*gravity acceleration*/
#define L 4.8 /*wave length*/
#define d 1.6 /*water depth*/
#define T 1.456/*effective wave period (include doppler effect)*/

/*DEFINE_PROFILE: define an inlet velocity profile  that varies as a function of z coordinates or t.*/
DEFINE_PROFILE(vertical_velocity,ft,var) /* DEFINE Macros, ft is a thread; var:index */
{
    /*define variables*/
    real r[ND_ND]; /*Coordinates, r[0] mean x coordinates, r[1] means y coordinates; r[2] is z coordinates*/
    real k; /*wave number*/
    real z; /*z(vertical) axis*/
    real omega; /*effective wave angular velocity*/
    real t; /*t*/
    k = 2.0*pi/L;     /*assign values to variables*/
    omega=2.0*pi/T;
    t = CURRENT_TIME; /* Special Fluent macro, current running t */

    face_t f; /* f is a face index for each face on the boundary */
   
    begin_f_loop(f,ft)/* face loop macro ,loop over all faces in a given face thread "ft" */
    {       
         F_CENTROID(r,f,ft); /*F_CENTROID finds the coordinate position of the centroid of the face "f" and stores the coordinates in the "r" array */
        z =r[1]; /* r[1] means y coordinates */
       
F_PROFILE(f,ft,var) = H*g*k*sinh(k*(z-0.782+d))*sin(-omega*t)/(2.0*(omega-k*U)*cosh(k*d)) + 3.0*H*H*(omega-k*U)*k*sinh(2.0*k*(z-0.782+d))*sin(-2.0*omega*t)/(16.0*pow(sinh(k*d),4.0));
/*   in the Fluent mode, free surface level is z=0.782m, positive z is in the gravity direction,however, z=0 is the mean free surface level, and z is negative gravity direction in the theory of wave */
   
/* verticla velocity: w_r= 2.0*a*g*k*sinh(k*(z+d))*sin(-omega*t)/(2.0*(omega-k*U)*cosh(k*d)) + 3.0*2.0*a*2.0*a*(omega-k*U)*k*sinh(2.0*k*(z+d))*sin(-2.0*omega*t)/(16.0*pow(sinh(k*d),4.0)); */
    }
    end_f_loop(f,ft)
}