NekInterface.h

Go to the documentation of this file.

/********************************************************************/
/*                  SOFTWARE COPYRIGHT NOTIFICATION                 */
/*                             Cardinal                             */
/*                                                                  */
/*                  (c) 2021 UChicago Argonne, LLC                  */
/*                        ALL RIGHTS RESERVED                       */
/*                                                                  */
/*                 Prepared by UChicago Argonne, LLC                */
/*               Under Contract No. DE-AC02-06CH11357               */
/*                With the U. S. Department of Energy               */
/*                                                                  */
/*             Prepared by Battelle Energy Alliance, LLC            */
/*               Under Contract No. DE-AC07-05ID14517               */
/*                With the U. S. Department of Energy               */
/*                                                                  */
/*                 See LICENSE for full restrictions                */
/********************************************************************/

#pragma once

#include "CardinalEnums.h"
#include "MooseTypes.h"
#include "NekBoundaryCoupling.h"
#include "NekVolumeCoupling.h"
#include "Function.h"

#include "inipp.hpp"
#include "nekrs.hpp"
#include "nrs.hpp"
#include "udf.hpp"
#include "inipp.hpp"
#include "mesh.h"

#include "libmesh/point.h"

#include <string>
#include <vector>

namespace nekrs
{

static int build_only;

int Nscalar();

void initializeHostMeshParameters();

void updateHostMeshParameters();

dfloat * host_x();
dfloat * host_y();
dfloat * host_z();

nrs_t * nrsPtr();

dfloat * getSgeo();
dfloat * getVgeo();

void checkFieldValidity(const field::NekFieldEnum & field);
void checkFieldValidity(const field::NekWriteEnum & field);

void copyDeviceToHost();

void computeWallDistance(const std::vector<int> & boundary_id);

std::vector<dfloat> yPlus(const std::vector<int> & boundary_id);

std::vector<dfloat> viscousDrag(const std::vector<int> & boundary);

void setAbsoluteTol(double tol);

Real scratchUnits(const int slot);

void nondimensional(const bool n);

void setRelativeTol(double tol);

void setNekSetupTime(const double & time);

double getNekSetupTime();

void setStartTime(const double & start);

bool isInitialized();

void write_usrwrk_field_file(const int & size,
                             const int & index,
                             const int & slot,
                             const std::string & prefix,
                             const dfloat & time,
                             const int & step,
                             const bool & write_coords);

void write_field_file(const std::string & prefix, const dfloat time, const int & step);

void buildOnly(int buildOnly);

int buildOnly();

void interpolateVolumeHex3D(const double * I, double * x, int N, double * Ix, int M);

bool hasCHT();

bool hasMovingMesh();

bool hasVariableDt();

bool hasBlendingSolver();

bool hasUserMeshSolver();

bool endControlElapsedTime();

bool endControlTime();

bool endControlNumSteps();

int scalarFieldOffset();

int velocityFieldOffset();

int fieldOffset();

mesh_t * entireMesh();

mesh_t * flowMesh();

mesh_t * temperatureMesh();

mesh_t * getMesh(const nek_mesh::NekMeshEnum pp_mesh);

int commRank();

int commSize();

bool hasTemperatureVariable();

bool hasTemperatureSolve();

bool hasScalarVariable(int scalarId);

bool hasHeatSourceKernel();

bool scratchAvailable();

void initializeScratch(const unsigned int & n_slots);

void freeScratch();

double viscosity();

double Pr();

void copyDeformationToDevice();

template <typename T>
void allgatherv(const std::vector<int> & base_counts,
                const T * input,
                T * output,
                const int multiplier = 1);

void displacementAndCounts(const std::vector<int> & base_counts,
                           int * counts,
                           int * displacement,
                           const int multiplier);

void interpolationMatrix(double * I, int starting_points, int ending_points);

void interpolateSurfaceFaceHex3D(
    double * scratch, const double * I, double * x, int N, double * Ix, int M);

Point centroidFace(int local_elem_id, int local_face_id);

Point centroid(int local_elem_id);

Point gllPoint(int local_elem_id, int local_node_id);

Point gllPointFace(int local_elem_id, int local_face_id, int local_node_id);

std::vector<double> usrwrkSideIntegral(const unsigned int & slot,
                                       const std::vector<int> & boundary,
                                       const nek_mesh::NekMeshEnum pp_mesh);

double usrwrkVolumeIntegral(const unsigned int & slot, const nek_mesh::NekMeshEnum pp_mesh);

void scaleUsrwrk(const unsigned int & slot, const dfloat & value);

double area(const std::vector<int> & boundary_id, const nek_mesh::NekMeshEnum pp_mesh);

double sideIntegral(const std::vector<int> & boundary_id, const field::NekFieldEnum & integrand,
                    const nek_mesh::NekMeshEnum pp_mesh);

double volume(const nek_mesh::NekMeshEnum pp_mesh);

void dimensionalizeVolume(double & integral);

void dimensionalizeArea(double & integral);

void dimensionalizeVolumeIntegral(const field::NekFieldEnum & integrand,
                                  const Real & volume,
                                  double & integral);

void dimensionalizeSideIntegral(const field::NekFieldEnum & integrand,
                                const Real & area,
                                double & integral);

void dimensionalizeSideIntegral(const field::NekFieldEnum & integrand,
                                const std::vector<int> & boundary_id,
                                double & integral,
                                const nek_mesh::NekMeshEnum pp_mesh);

double functionVolumeIntegral(const field::NekFieldEnum & integrand,
                              const double & volume,
                              const nek_mesh::NekMeshEnum pp_mesh,
                              const Function * function,
                              const Real & time);

double volumeIntegral(const field::NekFieldEnum & integrand,
                      const double & volume,
                      const nek_mesh::NekMeshEnum pp_mesh,
                      const Function * function,
                      const Real & time);

double evaluateFunctionOnMesh(const Function * f, const Real time, const int id);

double volumeNorm(const field::NekFieldEnum & integrand,
                  const nek_mesh::NekMeshEnum pp_mesh,
                  const Function * function,
                  const Real & time,
                  const Real & N);

double massFlowrate(const std::vector<int> & boundary_id,
                    const nek_mesh::NekMeshEnum pp_mesh);

double sideMassFluxWeightedIntegral(const std::vector<int> & boundary_id,
                                    const field::NekFieldEnum & integrand,
                                    const nek_mesh::NekMeshEnum pp_mesh);

double pressureSurfaceForce(const std::vector<int> & boundary_id, const Point & direction, const nek_mesh::NekMeshEnum pp_mesh);

double heatFluxIntegral(const std::vector<int> & boundary_id,
                        const nek_mesh::NekMeshEnum pp_mesh);

void limitTemperature(const double * min_T, const double * max_T);

void gradient(const int offset,
              const int e,
              const double * f,
              double * grad_f,
              const nek_mesh::NekMeshEnum pp_mesh);

double volumeExtremeValue(const field::NekFieldEnum & field,
                          const nek_mesh::NekMeshEnum pp_mesh,
                          const Function * function,
                          const Real & time,
                          const bool max);

double sideExtremeValue(const std::vector<int> & boundary_id, const field::NekFieldEnum & field,
                        const nek_mesh::NekMeshEnum pp_mesh, const bool max);

int Nfaces();

bool isHeatFluxBoundary(const int boundary);

bool isMovingMeshBoundary(const int boundary);

bool isTemperatureBoundary(const int boundary);

const std::string temperatureBoundaryType(const int boundary);

int polynomialOrder();

int Nelements();

int dim();

int NfaceVertices();

int NboundaryFaces();

int NboundaryID();

bool
validBoundaryIDs(const std::vector<int> & boundary_id, int & first_invalid_id, int & n_boundaries);

void storeBoundaryCoupling(const std::vector<int> & boundary_id, int & N);

struct characteristicScales
{
  double U_ref = 1;
  double T_ref = 0;
  double dT_ref = 1;
  double P_ref = 1;
  double L_ref = 1;
  double A_ref = 1;
  double V_ref = 1;
  double rho_ref = 1;
  double Cp_ref = 1;
  double flux_ref = 1;
  double source_ref = 1;
  double t_ref = 1;
  double s01_ref = 0;
  double ds01_ref = 1;
  double s02_ref = 0;
  double ds02_ref = 1;
  double s03_ref = 0;
  double ds03_ref = 1;
};

double (*solutionPointer(const field::NekFieldEnum & field))(int, int);
double (*solutionPointer(const field::NekWriteEnum & field))(int, int);

int scalarSlot(const int id);

double get_scalar01(const int id, const int surf_offset);

double get_scalar02(const int id, const int surf_offset);

double get_scalar03(const int id, const int surf_offset);

double get_usrwrk00(const int id, const int surf_offset);

double get_usrwrk01(const int id, const int surf_offset);

double get_usrwrk02(const int id, const int surf_offset);

double get_temperature(const int id, const int surf_offset);

double get_pressure(const int id, const int surf_offset);

double get_zero(const int id, const int surf_offset);

double get_unity(const int id, const int surf_offset);

double get_velocity_x(const int id, const int surf_offset);

double get_velocity_y(const int id, const int surf_offset);

double get_velocity_z(const int id, const int surf_offset);

double get_velocity(const int id, const int surf_offset);

double get_velocity_x_squared(const int id, const int surf_offset);

double get_velocity_y_squared(const int id, const int surf_offset);

double get_velocity_z_squared(const int id, const int surf_offset);

void initializeDimensionalScales(const double U,
                                 const double T,
                                 const double dT,
                                 const double L,
                                 const double rho,
                                 const double Cp,
                                 const double s01,
                                 const double ds01,
                                 const double s02,
                                 const double ds02,
                                 const double s03,
                                 const double ds03);

Real nondimensionalDivisor(const field::NekFieldEnum & field);
Real nondimensionalDivisor(const field::NekWriteEnum & field);

Real nondimensionalAdditive(const field::NekFieldEnum & field);
Real nondimensionalAdditive(const field::NekWriteEnum & field);

double referenceLength();

double referenceTime();

double referenceArea();

double referenceVolume();

// useful concept from Stack Overflow for templating MPI calls
template <typename T>
MPI_Datatype resolveType();

template <typename T>
void
allgatherv(const std::vector<int> & base_counts, const T * input, T * output, const int multiplier)
{
  int * recvCounts = (int *)calloc(commSize(), sizeof(int));
  int * displacement = (int *)calloc(commSize(), sizeof(int));
  displacementAndCounts(base_counts, recvCounts, displacement, multiplier);

  MPI_Allgatherv(input,
                 recvCounts[commRank()],
                 resolveType<T>(),
                 output,
                 (const int *)recvCounts,
                 (const int *)displacement,
                 resolveType<T>(),
                 platform->comm.mpiComm());

  free(recvCounts);
  free(displacement);
}

void initializeNekHostArrays();

// Accessors for NekRS host arrays
std::tuple<dfloat *, dfloat *, dfloat *> host_xyz();
std::vector<dfloat> & host_U();
std::vector<dfloat> & host_P();
std::vector<dfloat> & host_S();
dfloat * host_wrk();

mesh_t * createMesh2(mesh_t * _mesh, int Nc);

} // end namespace nekrs