Stochastic Tools Requirements Traceability Matrix

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'Monte Carlo;'
[]

[lhs]
  type = CSVDiff
  input = lhs.i
  csvdiff = 'lhs_csv_study_results_0001.csv'
  detail = 'latin hypercube;'
[]

[cartesian_product]
  type = CSVDiff
  input = cartesian_product.i
  csvdiff = 'cartesian_product_csv_study_results_0001.csv'
  detail = 'cartesian product;'
[]

[csv]
  type = CSVDiff
  input = csv.i
  cli_args = 'Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'CSV;'
[]

[csv_indices]
  type = CSVDiff
  input = csv.i
  cli_args = 'ParameterStudy/csv_column_indices=\'0 1\' Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'CSV when specifiying column indices;'
[]

[csv_names]
  type = CSVDiff
  input = csv.i
  cli_args = 'ParameterStudy/csv_column_names=\'study_sampler_0 study_sampler_1\' Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'CSV when specifiying column names;'
[]

[input_matrix]
  type = CSVDiff
  input = input_matrix.i
  cli_args = 'Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'input matrix;'
[]

[monte_carlo_num_samples]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=monte-carlo ParameterStudy/distributions=uniform'
  expect_err = 'The num_samples parameter is required to build the requested sampling type\.'
  detail = '\'num_samples\' parameter for Monte Carlo sampling;'
[]

[monte_carlo_distributions]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=monte-carlo ParameterStudy/num_samples=1'
  expect_err = 'The distributions parameter is required to build the requested sampling type\.'
  detail = '\'distributions\' parameter for Monte Carlo sampling;'
[]

[monte_carlo_unused]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=monte-carlo
                  ParameterStudy/num_samples=1
                  ParameterStudy/distributions=uniform
                  ParameterStudy/linear_space_items=\'0 1 1\''
  expect_err = 'The following parameters are unused for the selected sampling type:'
  detail = '\'linear_space_items\' parameter with Monte Carlo sampling;'
[]

[lhs_num_samples]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=lhs ParameterStudy/distributions=uniform'
  expect_err = 'The num_samples parameter is required to build the requested sampling type\.'
  detail = '\'num_samples\' parameter for latin hypercube sampling;'
[]

[lhs_distributions]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=lhs ParameterStudy/num_samples=1'
  expect_err = 'The distributions parameter is required to build the requested sampling type\.'
  detail = '\'distributions\' parameter for latin hypercube sampling;'
[]

[cartesian_product]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=cartesian-product'
  expect_err = 'The linear_space_items parameter is required to build the requested sampling type\.'
  detail = '\'linear_space_items\' parameter for cartesian product sampling;'
[]

[csv]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=csv'
  expect_err = 'The csv_samples_file parameter is required to build the requested sampling type\.'
  detail = '\'samples_file\' parameter for CSV sampling;'
[]

[csv_both]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=csv
                  ParameterStudy/csv_samples_file=gold/monte_carlo_csv_study_samples_0001.csv
                  ParameterStudy/csv_column_indices=1
                  ParameterStudy/csv_column_names=foo'
  expect_err = '\'csv_column_indices\' and \'csv_column_names\' cannot both be set\.'
  detail = 'both \'csv_column_indices\' and \'csv_column_names\' parameters are specified for CSV sampling;'
[]

[input_matrix]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=input-matrix'
  expect_err = 'The input_matrix parameter is required to build the requested sampling type\.'
  detail = '\'matrix\' parameter for input matrix sampling;'
[]

[distributions]
  type = CSVDiff
  input = distributions.i
  csvdiff = distributions_csv_study_results_0001.csv
  requirement = 'The system shall be able to setup a Monte Carlo parameter study with uniform, normal, Weibull, lognormal, and truncated normal distributions.'
[]

[missing]
  type = RunException
  input = distributions_error.i
  cli_args = 'ParameterStudy/distributions=\'uniform uniform\'
                  ParameterStudy/uniform_upper_bound=\'1 1\''
  expect_err = 'The uniform_lower_bound parameter is required to build the listed distributions\.'
  detail = 'missing;'
[]

[inconsistent]
  type = RunException
  input = distributions_error.i
  cli_args = 'ParameterStudy/distributions=\'uniform uniform\'
                  ParameterStudy/uniform_lower_bound=\'0\'
                  ParameterStudy/uniform_upper_bound=\'1 1\''
  expect_err = 'The number of entries in uniform_lower_bound does not match the number of required entries \(\d+\) to build the listed distributions\.'
  detail = 'inconsitent size;'
[]

[unused]
  type = RunException
  input = distributions_error.i
  cli_args = 'ParameterStudy/distributions=\'uniform uniform\'
                  ParameterStudy/uniform_lower_bound=\'0 0\'
                  ParameterStudy/uniform_upper_bound=\'1 1\'
                  ParameterStudy/normal_mean=1'
  expect_err = 'The following parameters are unused for the listed distributions:'
  detail = 'unused;'
[]

[true]
  type = JSONDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=json
                  ParameterStudy/show_study_objects=true
                  --color off'
  jsondiff = monte_carlo_json.json
  rel_err = 1e-4
  expect_out = 'Type:\s+StatisticsReporter'
  detail = 'compute statistics;'
[]

[false]
  type = RunApp
  input = monte_carlo.i
  cli_args = 'ParameterStudy/compute_statistics=false
                  ParameterStudy/show_study_objects=true
                  --color off'
  absent_out = 'Type:\s+StatisticsReporter'
  detail = 'not compute statistics;'
[]

[unused]
  type = RunException
  input = monte_carlo.i
  cli_args = 'ParameterStudy/compute_statistics=false
                  ParameterStudy/statistics=mean
                  ParameterStudy/ci_levels=0.1
                  ParameterStudy/ci_replicates=10'
  expect_err = 'The following parameters are unused since statistics are not being computed: ci_levels, ci_replicates, statistics'
  detail = 'error if there are unused statistics parameters;'
[]

[none]
  type = RunApp
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=none
                  ParameterStudy/compute_statistics=false
                  ParameterStudy/show_study_objects=true
                  --color off'
  expect_out = 'outputs : none'
  absent_out = 'Base Type:\s+Output'
  detail = 'no output;'
[]

[csv]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=csv
                  ParameterStudy/compute_statistics=false
                  ParameterStudy/show_study_objects=true
                  --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'Type:\s+CSV'
  absent_out = 'Type:\s+JSON'
  detail = 'CSV output;'
[]

[json]
  type = JSONDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=json
                  ParameterStudy/show_study_objects=true
                  --color off'
  jsondiff = monte_carlo_json.json
  rel_err = 1e-4
  expect_out = 'Type:\s+JSON'
  absent_out = 'Type:\s+CSV'
  detail = 'JSON output;'
[]

[normal]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=normal --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : normal'
  detail = 'one sub-app per sample;'
[]

[batch_reset]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-reset --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-reset'
  detail = 'batches with one sub-app per processor while re-initializing;'
[]

[batch_restore]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-restore --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore'
  detail = 'batches one sub-app per processor while restoring;'
[]

[batch_keep_solution]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-keep-solution --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'keep_solution_during_restore : 1\n.*\n\s*mode : batch-restore'
  detail = 'batches one sub-app per processor while restoring and keeping solution;'
[]

[batch_no_restore]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-no-restore --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore\n\s*no_restore : 1'
  detail = 'batches one sub-app per processor without restoring;'
[]

[parsed]
  type = CSVDiff
  input = distributions.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = distributions_csv_study_results_0001.csv
  expect_out = 'mode : batch-reset'
  detail = 'parameters are parsed;'
[]

[not_controllable]
  type = CSVDiff
  input = monte_carlo_not_controllable.i
  cli_args = 'Outputs/file_base=monte_carlo_csv ParameterStudy/sampler_column_names=\'BCs_left_value BCs_right_value\' ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-reset'
  detail = 'parameters are not controllable;'
[]

[transient]
  type = CSVDiff
  input = monte_carlo_transient.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_transient_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore'
  detail = 'parameters are controllable in a transient problem;'
[]

[pseudo_transient]
  type = CSVDiff
  input = monte_carlo_pseudo_transient.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_pseudo_transient_csv_study_results_0001.csv'
  expect_out = 'keep_solution_during_restore : 1\n.*\n\s*mode : batch-restore'
  detail = 'parameters are controllable in a pseudo-transient problem;'
[]

[steady]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore\n\s*no_restore : 1'
  detail = 'parameters are controllable in a steady-state problem;'
[]

[eigen]
  type = CSVDiff
  input = monte_carlo_eigen.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_eigen_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore\n\s*no_restore : 1'
  detail = 'parameters are controllable in a eigenvalue problem;'
[]

[create]
  type = CheckFiles
  input = surrogate.i
  check_files = 'surrogate_trainer_poly_regression.rd/data'
  requirement = 'The system shall be able to create a surrogate that can be read in by file and use in aux kernels.'
[]

[var]
  type = RunApp
  input = surrogate_aux.i
  prereq = create
  detail = 'standard field variables and'
[]

[array_var]
  type = RunApp
  input = surrogate_array_aux.i
  prereq = create
  detail = 'array field variables.'
[]

[parameter_unspecified]
  type = RunException
  input = surrogate_aux.i
  cli_args = 'AuxKernels/u_aux/parameters=\'funx 3.14 var pp foo\''
  prereq = create
  expect_err = '\'foo\' is not listed in \'scalar_parameters\'\, \'coupled_variables\'\, or \'coupled_array_variables\'.'
  detail = 'a parameter is undefined,'
[]

[scalar_missing]
  type = RunException
  input = surrogate_aux.i
  cli_args = 'AuxKernels/u_aux/parameters=\'funx 3.14 var pp foo\' AuxKernels/u_aux/scalar_parameters=\'funx funy pp foo\''
  prereq = create
  expect_err = '\'foo\' is not a postprocessor or a function.'
  detail = 'a scalar parameter is not found, or'
[]

[wrong_array_size]
  type = RunException
  input = surrogate_array_aux.i
  cli_args = 'AuxVariables/u/components=3'
  prereq = create
  expect_err = 'The number of components in \'\w+\' \(\d+\) does not match the number of components in \'\w+\' \(\d+\).'
  detail = 'a coupled array variable does not match the number of components.'
[]

[scalar_surrogate_aux]
  type = 'CSVDiff'
  input = 'surrogate_scalar_aux.i'
  csvdiff = 'surrogate_scalar_aux_out.csv'
  abs_zero = 1e-06
  group = 'simple'
  recover = false
  requirement = 'The system shall be able to evaluate a surrogate at given scalar variable values and insert the result into a scalar variable'
[]

[scalar_surrogate_aux_pp_arg]
  type = 'CSVDiff'
  input = 'surrogate_scalar_aux.i'
  csvdiff = 'scalar_surrogate_aux_pp_arg.csv'
  abs_zero = 1e-06
  group = 'simple'
  recover = false
  cli_args = 'Outputs/file_base=scalar_surrogate_aux_pp_arg AuxScalarKernels/scalar_eval/parameters="Tpp 5"'
  requirement = 'The system shall be able to evaluate a surrogate at given postprocessor values and insert the result into a scalar variable'
[]

[read-parameters]
  type = CSVDiff
  input = libtorch_drl_control.i
  csvdiff = parameter_read.csv
  cli_args = "Outputs/file_base='parameter_read'"
  requirement = "The system shall be able to read a neural network from a parameter file and use it to control a transient process."
  libtorch = true
[]

[without-nn]
  type = CSVDiff
  input = libtorch_drl_control.i
  csvdiff = run_wo_nn.csv
  cli_args = "Outputs/file_base='run_wo_nn' Controls/inactive='src_control' Postprocessors/left_flux/control_name='src_control_empty' Postprocessors/log_prob_left_flux/control_name='src_control_empty'"
  requirement = "The system shall be able to run with a DRL controller without having a neural net initialized in it."
  libtorch = true
[]

[uniform]
  type = CSVDiff
  input = uniform.i
  csvdiff = 'uniform_out.csv'
  allow_test_objects = true

  detail = 'uniform,'
[]

[weibull]
  type = CSVDiff
  input = weibull.i
  csvdiff = 'weibull_out.csv'
  allow_test_objects = true

  detail = 'Weibull (3 parameter),'
[]

[kernel1d_gaussian]
  type = CSVDiff
  input = kernel_density_1d_gaussian.i
  csvdiff = 'kernel_density_1d_gaussian_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and data file as input,'
[]

[kernel1d_uniform]
  type = CSVDiff
  input = kernel_density_1d_uniform.i
  csvdiff = 'kernel_density_1d_uniform_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Uniform kernel and data file as input,'
[]

[kernel1d_datvec]
  type = CSVDiff
  input = kernel_density_1d_datvec.i
  csvdiff = 'kernel_density_1d_datvec_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and data vector as input,'
[]

[kernel1d_userbw]
  type = CSVDiff
  input = kernel_density_1d_userbw.i
  csvdiff = 'kernel_density_1d_userbw_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and user defined bandwidth,'
[]

[kernel1d_sdbw]
  type = CSVDiff
  input = kernel_density_1d_sdbw.i
  csvdiff = 'kernel_density_1d_sdbw_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and standard deviation as bandwidth,'
[]

[normal]
  type = CSVDiff
  input = normal.i
  csvdiff = 'normal_out.csv'
  allow_test_objects = true

  detail = 'normal,'
[]

[truncated_normal]
  type = CSVDiff
  input = truncated_normal.i
  csvdiff = 'truncated_normal_out.csv'
  allow_test_objects = true

  detail = 'truncated normal,'
[]

[lognormal]
  type = CSVDiff
  input = lognormal.i
  csvdiff = 'lognormal_out.csv'
  allow_test_objects = true

  detail = 'lognormal,'
[]

[gamma]
  type = CSVDiff
  input = gamma.i
  csvdiff = 'gamma_out.csv'
  allow_test_objects = true

  detail = 'gamma,'
[]

[beta]
  type = CSVDiff
  input = beta.i
  csvdiff = 'beta_out.csv'
  allow_test_objects = true

  detail = 'beta,'
[]

[fdist]
  type = CSVDiff
  input = fdist.i
  csvdiff = 'fdist_out.csv'
  allow_test_objects = true

  detail = 'F,'
[]

[studentt]
  type = CSVDiff
  input = studentt.i
  csvdiff = 'studentt_out.csv'
  allow_test_objects = true

  detail = 'student t,'
[]

[johnsonsb]
  type = CSVDiff
  input = johnsonsb.i
  csvdiff = 'johnsonsb_out.csv'
  allow_test_objects = true

  detail = 'Johnson Special Bounded (SB), and'
[]

[logistic]
  type = CSVDiff
  input = logistic.i
  csvdiff = 'logistic_out.csv'
  allow_test_objects = true

  detail = 'logistic distributions.'
[]

[normal_direct]
  type = CSVDiff
  input = normal_direct.i
  csvdiff = 'normal_direct_out.csv'
  allow_test_objects = true
  requirement = 'The system shall provide a normal distribution with the ability to directly call methods with distribution inputs.'
  design = 'Normal.md'
  issues = '#12720'
[]

[normal_direct_type_error]
  type = RunException
  input = normal_direct_type_error.i
  allow_test_objects = true
  expect_err = "The 'cdf' object failed to retrieve 'this_is_the_wrong_type' distribution with the desired type."
  requirement = 'The system shall produce an error if a distribution is retrieved with the differing type than supplied.'
  design = 'Normal.md'
  issues = '#12720'
[]

[libtorch_drl_reward_linear]
  type = CSVDiff
  input = drl_reward.i
  csvdiff = drl_reward_linear.csv
  requirement = "The system shall be able to evaluate a linear DRL reward function with a given, time-dependent target."
  issues = '#19571'
  design = 'ScaledAbsDifferenceDRLRewardFunction.md'
  cli_args = "Outputs/file_base='drl_reward_linear'"
  libtorch = true
[]

[generate]
  type = 'CSVDiff'
  input = 'random_ic_distribution_test.i'
  csvdiff = 'random_ic_distribution_test_initial_histo_0000.csv'
  mesh_mode = 'replicated' # sensitive to difference in partitioning

  requirement = 'The system shall generate parallel agnostic random initial conditions using a distribution function.'
[]

[test_err_distribution_and_min_max]
  type = 'RunException'
  input = 'random_ic_distribution_test.i'
  cli_args = 'ICs/u_aux/min=0 ICs/u_aux/max=1'
  prereq = 'generate'
  expect_err = "Cannot use together with 'min' or 'max' parameter"

  requirement = 'The system shall generate an error the random initial condition is used with both a distribution and min or max value defined.'
[]

[loglikelihood_scalar]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'loglikelihood_scalar.json'
  min_parallel = 2
  max_parallel = 2
  detail = 'using a Gaussian type with log-likelihoods requested and model/experiment values are scalars; '
[]

[likelihood_scalar]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'likelihood_scalar.json'
  cli_args = "Likelihood/gaussian/log_likelihood='false' Outputs/file_base='likelihood_scalar'"
  min_parallel = 2
  max_parallel = 2
  detail = 'using a Gaussian type with likelihoods requested and model/experiment values are scalars; '
[]

[truncatedgaussian]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'truncatedgaussian.json'
  cli_args = "Likelihood/gaussian/type='TruncatedGaussian' Likelihood/gaussian/lower_bound='-1.0' Likelihood/gaussian/upper_bound='1.0' Outputs/file_base='truncatedgaussian'"
  min_parallel = 2
  max_parallel = 2
  detail = 'using a truncated Gaussian type with likelihoods requested and model/experiment values are scalars. '
[]

[extremevalue]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'extremevalue.json'
  cli_args = "Likelihood/gaussian/type='ExtremeValue' Outputs/file_base='extremevalue'"
  min_parallel = 2
  max_parallel = 2
  detail = 'using a Extreme Value type with likelihoods requested and model/experiment values are scalars. '
[]

[truncatedgaussian_bounds]
  type = RunException
  input = main.i
  allow_test_objects = True
  cli_args = "Likelihood/gaussian/type=TruncatedGaussian Likelihood/gaussian/lower_bound=1.0 Likelihood/gaussian/upper_bound=-1.0"
  detail = 'the specified lower bound is greater than the upper bound; '
  expect_err = "The specified lower bound should be less than the upper bound."
[]

[single_normal]
  type = CSVDiff
  input = parent_single.i
  cli_args = MultiApps/sub/mode=normal
  csvdiff = 'parent_single_out_storage_0002.csv'

  detail = 'for a single parameter, '
[]

[single_batch]
  type = CSVDiff
  input = parent_single.i
  csvdiff = 'parent_single_out_storage_0002.csv'
  prereq = parent/single_normal
  max_parallel = 3

  detail = 'for a single parameter for a batch of sub-applications,'
[]

[multiple_normal]
  type = CSVDiff
  input = parent_multiple.i
  cli_args = MultiApps/sub/mode=normal
  csvdiff = 'parent_multiple_out_storage_0002.csv'
  prereq = parent/single_batch

  detail = 'for multiple parameters, and'
[]

[vector]
  type = CSVDiff
  input = parent_vector.i
  csvdiff = 'parent_vector_out_storage_0002.csv parent_vector_out_prop_A_0002.csv parent_vector_out_prop_B_0002.csv parent_vector_out_prop_C_0002.csv parent_vector_out_prop_D_0002.csv'
  max_parallel = 3

  detail = 'for vector parameters for a batch of sub-applications,'
[]

[multiple_batch]
  type = CSVDiff
  input = parent_multiple.i
  csvdiff = 'parent_multiple_out_storage_0002.csv'
  prereq = parent/multiple_normal
  max_parallel = 3

  detail = 'for multiple parameters for a batch of sub-applications.'
[]

[multiple_batch_with_index]
  type = CSVDiff
  input = parent_multiple.i
  csvdiff = 'parent_multiple_out_storage_0002.csv'
  cli_args = "Controls/cmdline/param_names='Mesh/ymax[1] Mesh/xmax[0]'"
  prereq = parent/multiple_normal
  max_parallel = 3

  detail = 'for multiple parameters using their global column indexes for a batch of sub-applications.'
[]

[wrong_batch_mode]
  type = RunException
  input = parent_single.i
  cli_args = MultiApps/sub/mode=batch-restore
  expect_err = "The MultiApp object, 'sub', supplied to the 'cmdline' object is setup to run in 'batch-restore' mode, when using this mode command line arguments cannot be modified; batch-reset mode should be used instead."
  requirement = "The system shall report an error when the supplied sampler object operates in a mode that does not allow for command line arguments to be modified."
  prereq = parent/multiple_batch
  max_parallel = 3
[]

[wrong_execute_on]
  type = RunException
  input = parent_single.i
  cli_args = Samplers/sample/execute_on=INITIAL
  expect_err = "The sampler object, 'sample', is being used by the 'cmdline' object, thus the 'execute_on' of the sampler must include 'PRE_MULTIAPP_SETUP'."
  requirement = "The system shall report an error when the supplied sampler does not use the correct execution flags."
  prereq = parent/multiple_batch
  max_parallel = 3
[]

[vector_no_bracket]
  type = RunException
  input = parent_vector.i
  cli_args = "Controls/cmdline/param_names='Mesh/xmax[0] Materials/const/prop_values[1,(1.5),2,2] Mesh/ymax'"
  expect_err = "If the bracket is used, it must be provided to every parameter."
  requirement = "The system shall report an error when '[]' syntax is not used for all parameters."
  max_parallel = 3
[]

[vector_out_of_bound]
  type = RunException
  input = parent_vector.i
  cli_args = "Controls/cmdline/param_names='Mesh/xmax[0] Materials/const/prop_values[1,(1.5),2,2] Mesh/ymax[4]'"
  expect_err = "The provided global column index \(4\) for Mesh/ymax is out of bound."
  requirement = "The system shall report an error when provided global column index is out of bound."
  max_parallel = 3
[]

[normal]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  cli_args = MultiApps/runner/mode=normal
  requirement = "The system shall support pulling postprocessor data from a sub-application for each row of sampled data."
[]

[batch]
  type = CheckFiles
  input = parent_full_solve.i
  check_files = 'parent_full_solve_out_runner0.e parent_full_solve_out_runner1.e parent_full_solve_out_runner2.e parent_full_solve_out_runner3.e parent_full_solve_out_runner4.e'
  requirement = "The system shall support running sub-applications in batches."
[]

[n1]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  max_parallel = 1

  detail = 'on a single processor,'
[]

[n2]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  max_parallel = 2
  min_parallel = 2

  detail = 'on multiple processors, and'
[]

[n2_restore]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  max_parallel = 2
  min_parallel = 2
  cli_args = "MultiApps/runner/mode=batch-restore"

  detail = 'on multiple processors using in-memory backup.'
[]

[normal]
  type = CSVDiff
  input = parent_transient.i
  cli_args = MultiApps/runner/mode=normal
  csvdiff = 'parent_transient_out_storage_0000.csv parent_transient_out_storage_0001.csv parent_transient_out_storage_0002.csv parent_transient_out_storage_0003.csv'

  detail = 'with individual sub-applications,'
[]

[batch]
  type = CSVDiff
  input = parent_transient.i
  csvdiff = 'parent_transient_out_storage_0000.csv parent_transient_out_storage_0001.csv parent_transient_out_storage_0002.csv parent_transient_out_storage_0003.csv'
  prereq = transient/normal
  max_parallel = 5

  detail = 'with sub-applications batches using in-memory restore functionality.'
[]

[mode_error]
  type = RunException
  input = parent_transient.i
  cli_args = MultiApps/runner/mode=batch-reset
  expect_err = "The supplied mode, 'batch-reset', currently"
  requirement = "The SamplerTransientMultiApp object shall error if the 'batch-reset' mode is supplied."
[]

[normal]
  type = CSVDiff
  input = parent_transient.i
  cli_args = 'MultiApps/runner/mode=normal runner:Outputs/csv=true'
  csvdiff = 'parent_transient_out_runner0.csv
                 parent_transient_out_runner1.csv
                 parent_transient_out_runner2.csv
                 parent_transient_out_runner3.csv
                 parent_transient_out_runner4.csv'
  detail = 'individual sub-applications;'
[]

[batch]
  type = CSVDiff
  input = parent_transient.i
  cli_args = 'MultiApps/runner/mode=batch-restore runner:Outputs/csv=true'
  csvdiff = 'parent_transient_out_runner0.csv
                 parent_transient_out_runner1.csv
                 parent_transient_out_runner2.csv
                 parent_transient_out_runner3.csv
                 parent_transient_out_runner4.csv'
  detail = 'sub-application batches;'
[]

[single]
  type = CSVDiff
  input = parent_single.i
  csvdiff = 'parent_single_out_sub0.csv parent_single_out_sub1.csv parent_single_out_sub2.csv'

  detail = 'for a single parameter and'
[]

[multiple]
  type = CSVDiff
  input = parent_multiple.i
  csvdiff = 'parent_multiple_out_sub0.csv parent_multiple_out_sub1.csv parent_multiple_out_sub2.csv'

  detail = 'for a multiple parameters.'
[]

[wrong_multiapp_type]
  type = RunException
  input = parent_wrong_multiapp_type.i
  expect_err = "The MultiApp must construct its sub-apps in initial setup but not during its creation"

  requirement = "The system shall report an error when sub-apps are constructed too early thus unable to be changed by samplers."
[]

[wrong_num_params]
  type = RunException
  input = parent_wrong_num_params.i
  expect_err = "The number of columns \(2\) must match the number of parameters \(3\)"

  requirement = "The system shall report an error when the number of samples differs from the number of command line parameters."
[]

[function]
  type = JSONDiff
  input = conditional_main.i
  jsondiff = 'conditional_main_out.json'
  detail = 'with a parsed function dependent on time and sampler values;'
[]

[error]
  type = RunException
  input = conditional_main.i
  cli_args = MultiApps/runner/mode=normal
  expect_err = "Conditionally run sampler multiapp only works in batch modes."
  detail = 'error if running in normal mode.'
[]

[normal]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  jsondiff = main_out.json

  detail = "normal execution,"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  cli_args = 'MultiApps/runner/mode=batch-restore'
  jsondiff = main_out.json

  detail = "batch execution with memory-based restoring, and"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  cli_args = 'MultiApps/runner/mode=batch-reset'
  jsondiff = main_out.json

  detail = "batch execution with reset-based restoring."
[]

[error]
  type = RunException
  input = main.i
  allow_test_objects = true
  expect_err = "The size of the sampler has changed; SamplerTransientMultiApp object do not support dynamic Sampler output."

  requirement = "The system shall report an error when the size of a sampler is altered an sub-applications are progressing with time with the main application."
[]

[normal]
  type = CSVDiff
  input = stm.i
  csvdiff = stm_out_storage_0001.csv
  detail = 'normal multiapp mode;'
[]

[batch]
  type = CSVDiff
  input = stm.i
  cli_args = 'MultiApps/main/mode=batch-reset'
  csvdiff = stm_out_storage_0001.csv
  detail = 'batch multiapp mode;'
[]

[normal]
  type = CSVDiff
  input = stm_batch.i
  cli_args = 'Outputs/file_base=stm_out'
  csvdiff = stm_out_storage_0001.csv
  detail = 'normal multiapp mode;'
[]

[batch_reset]
  type = CSVDiff
  input = stm_batch.i
  cli_args = 'Outputs/file_base=stm_out MultiApps/main/mode=batch-reset'
  csvdiff = stm_out_storage_0001.csv
  detail = 'batch-reset multiapp mode;'
[]

[batch_restore]
  type = CSVDiff
  input = stm_batch.i
  cli_args = 'Outputs/file_base=stm_out MultiApps/main/mode=batch-restore'
  csvdiff = stm_out_storage_0001.csv
  detail = 'batch-restore multiapp mode;'
[]

[normal]
  type = CSVDiff
  input = 'stm_small_batch.i'
  cli_args = 'MultiApps/main/mode=normal main:Outputs/csv=true main:sub:Outputs/csv=true main:sub:subsub:Outputs/csv=true'
  csvdiff = 'stm_small_batch_out_main0.csv stm_small_batch_out_main0_sub0.csv stm_small_batch_out_main0_sub0_subsub0.csv stm_small_batch_out_main0_sub0_subsub1.csv
                 stm_small_batch_out_main1.csv stm_small_batch_out_main1_sub0.csv stm_small_batch_out_main1_sub0_subsub0.csv stm_small_batch_out_main1_sub0_subsub1.csv
                 stm_small_batch_out_main2.csv stm_small_batch_out_main2_sub0.csv stm_small_batch_out_main2_sub0_subsub0.csv stm_small_batch_out_main2_sub0_subsub1.csv
                 stm_small_batch_out_main3.csv stm_small_batch_out_main3_sub0.csv stm_small_batch_out_main3_sub0_subsub0.csv stm_small_batch_out_main3_sub0_subsub1.csv'
  detail = 'normal multiapp mode;'
[]

[batch_reset]
  type = CSVDiff
  input = 'stm_small_batch.i'
  cli_args = 'MultiApps/main/mode=batch-reset main:Outputs/csv=true main:sub:Outputs/csv=true main:sub:subsub:Outputs/csv=true'
  csvdiff = 'stm_small_batch_out_main0.csv stm_small_batch_out_main0_sub0.csv stm_small_batch_out_main0_sub0_subsub0.csv stm_small_batch_out_main0_sub0_subsub1.csv
                 stm_small_batch_out_main1.csv stm_small_batch_out_main1_sub0.csv stm_small_batch_out_main1_sub0_subsub0.csv stm_small_batch_out_main1_sub0_subsub1.csv
                 stm_small_batch_out_main2.csv stm_small_batch_out_main2_sub0.csv stm_small_batch_out_main2_sub0_subsub0.csv stm_small_batch_out_main2_sub0_subsub1.csv
                 stm_small_batch_out_main3.csv stm_small_batch_out_main3_sub0.csv stm_small_batch_out_main3_sub0_subsub0.csv stm_small_batch_out_main3_sub0_subsub1.csv'
  detail = 'batch-reset multiapp mode;'
[]

[batch_restore]
  type = CSVDiff
  input = 'stm_small_batch.i'
  cli_args = 'MultiApps/main/mode=batch-restore main:Outputs/csv=true main:sub:Outputs/csv=true main:sub:subsub:Outputs/csv=true'
  csvdiff = 'stm_small_batch_out_main0.csv stm_small_batch_out_main0_sub0.csv stm_small_batch_out_main0_sub0_subsub0.csv stm_small_batch_out_main0_sub0_subsub1.csv
                 stm_small_batch_out_main1.csv stm_small_batch_out_main1_sub0.csv stm_small_batch_out_main1_sub0_subsub0.csv stm_small_batch_out_main1_sub0_subsub1.csv
                 stm_small_batch_out_main2.csv stm_small_batch_out_main2_sub0.csv stm_small_batch_out_main2_sub0_subsub0.csv stm_small_batch_out_main2_sub0_subsub1.csv
                 stm_small_batch_out_main3.csv stm_small_batch_out_main3_sub0.csv stm_small_batch_out_main3_sub0_subsub0.csv stm_small_batch_out_main3_sub0_subsub1.csv'
  detail = 'batch-restore multiapp mode;'
[]

[normal_less_procs]
  type = CSVDiff
  input = main.i
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  detail = 'with less processors than rows in normal mode;'
[]

[normal_more_procs]
  type = CSVDiff
  input = main.i
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  detail = 'with more processors than rows in normal mode;'
[]

[normal_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  detail = 'with specified minimum processors per app in normal mode;'
[]

[reset_less_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  detail = 'with less processors than rows in batch-reset mode;'
[]

[reset_more_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  detail = 'with more processors than rows in batch-reset mode;'
[]

[reset_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  detail = 'with specified minimum processors per app in batch-reset mode;'
[]

[restore_less_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  detail = 'with less processors than rows in batch-restore mode;'
[]

[restore_more_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  detail = 'with more processors than rows in batch-restore mode;'
[]

[restore_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2 Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  detail = 'with specified minimum processors per app in batch-restore mode;'
[]

[error_min_procs]
  type = RunException
  input = main.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=1'
  allow_test_objects = true
  expect_err = "Sampler and multiapp communicator configuration inconsistent. Please ensure that 'MultiApps/sub/min\(max\)_procs_per_app' and 'Samplers/sample/min\(max\)_procs_per_row' are the same."
  min_parallel = 2
  detail = 'error when partitionings do not match;'
[]

[normal_less_procs]
  type = CSVDiff
  input = main_transient.i
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  max_parallel = 5
  detail = 'with less processors than rows in normal mode;'
[]

[normal_more_procs]
  type = CSVDiff
  input = main_transient.i
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 6
  detail = 'with more processors than rows in normal mode;'
[]

[normal_min_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 4
  detail = 'with specified minimum processors per app in normal mode;'
[]

[restore_less_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  max_parallel = 5
  detail = 'with less processors than rows in batch-restore mode;'
[]

[restore_more_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 6
  detail = 'with more processors than rows in batch-restore mode;'
[]

[restore_min_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2 Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 4
  detail = 'with specified minimum processors per app in batch-restore mode;'
[]

[error_min_procs]
  type = RunException
  input = main_transient.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=1'
  allow_test_objects = true
  expect_err = "Sampler and multiapp communicator configuration inconsistent. Please ensure that 'MultiApps/sub/min\(max\)_procs_per_app' and 'Samplers/sample/min\(max\)_procs_per_row' are the same."
  min_parallel = 2
  detail = 'error when partitionings do not match;'
[]

[full_solve]
  type = CheckFiles
  input = parent_full_solve.i
  check_files = 'parent_full_solve_out_runner0.e parent_full_solve_out_runner1.e parent_full_solve_out_runner2.e parent_full_solve_out_runner3.e parent_full_solve_out_runner4.e'

  requirement = 'The system shall provide the ability to create a full-solve type sub-application from sampled data from distributions.'
[]

[transient]
  type = CheckFiles
  input = parent_transient.i
  check_files = 'parent_transient_out_runner0.e parent_transient_out_runner1.e parent_transient_out_runner2.e parent_transient_out_runner3.e parent_transient_out_runner4.e'

  requirement = 'The system shall provide the ability to create a transient sub-application from the sample data generated from distributions.'
[]

[transient_cmdline_control]
  type = Exodiff
  input = parent_transient_cmd_control.i
  exodiff = 'parent_transient_cmd_control_out_runner0.e parent_transient_cmd_control_out_runner1.e parent_transient_cmd_control_out_runner2.e parent_transient_cmd_control_out_runner3.e parent_transient_cmd_control_out_runner4.e'

  requirement = 'The system shall provide the ability to set a transient sub-application command line parameters from a sample distribution.'

  issues = '#17486 #17548'
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json

  detail = "using normal operation;"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  cli_args = MultiApps/runner/mode=batch-restore

  detail = "using in memory backup operation;"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  cli_args = MultiApps/runner/mode=batch-reset

  detail = "using reset operation."
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"

  detail = "using normal operation;"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "MultiApps/runner/mode=batch-restore Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"

  detail = "using in memory backup operation;"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "MultiApps/runner/mode=batch-reset Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"

  detail = "using reset operation."
[]

[cmdline_normal]
  type = CSVDiff
  input = main_full_solve.i
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'command line control in normal mode;'
[]

[cmdline_batch]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-reset"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'command line control in batch mode;'
[]

[transfer_normal]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in normal mode;'
[]

[transfer_batch_reset]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-reset Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in batch-reset mode;'
[]

[transfer_batch_restore]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-restore Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in batch-restore mode;'
[]

[cmdline]
  type = CSVDiff
  input = main_transient.i
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  detail = 'command line control;'
[]

[transfer_normal]
  type = CSVDiff
  input = main_transient.i
  cli_args = "Transfers/inactive='' Controls/active=''"
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  detail = 'parameter transfer in normal mode;'
[]

[transfer_batch]
  type = CSVDiff
  input = main_transient.i
  cli_args = "MultiApps/sub/mode=batch-restore Transfers/inactive='' Controls/active=''"
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  detail = 'parameter transfer in batch mode;'
[]

[ais_al_uval]
  type = 'JSONDiff'
  input = ais_al.i
  jsondiff = 'ais_al.json'
  detail = 'using the U learning function.'
[]

[SingleProc_SingleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  jsondiff = 'single_proc_single_row_ufunction.json'
  detail = 'a single row in the sampler using the U learning function;'
[]

[MultipleProc_MultipleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  min_parallel = 2
  jsondiff = 'multiple_proc_multiple_row_ufunction.json'
  cli_args = "Samplers/mc/num_batch=2 Outputs/file_base='multiple_proc_multiple_row_ufunction'"
  detail = 'multiple rows in the sampler using the U learning function and automatically increase the number of steps when re-training is needed;'
[]

[SingleProc_SingleRow_Ufunction_retraining]
  type = 'JSONDiff'
  input = main_adam.i
  jsondiff = 'single_proc_single_row_ufunction_retraining.json'
  cli_args = "Reporters/conditional/learning_function_threshold=5.0 Outputs/file_base='single_proc_single_row_ufunction_retraining'"
  detail = 'a single row in the sampler using the U learning function and automatically increase the number of steps when re-training is needed;'
[]

[COV_squared_exp]
  type = 'JSONDiff'
  input = main_adam.i
  jsondiff = 'cov_squared_exp.json'
  cli_args = "Reporters/conditional/learning_function='COV' Reporters/conditional/learning_function_threshold=0.025 Outputs/file_base='cov_squared_exp'"
  detail = 'a single row in the sampler using the COV learning function and automatically increase the number of steps when re-training is needed;'
[]

[SingleProc_SingleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  min_parallel = 1
  max_parallel = 1
  jsondiff = 'single_proc_single_row_ufunction.json'
  detail = 'a single row in the sampler using the U learning function;'
[]

[MultipleProc_MultipleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  min_parallel = 2
  max_parallel = 2
  jsondiff = 'multiple_proc_multiple_row_ufunction.json'
  cli_args = "Samplers/mc/num_batch=2 Outputs/file_base='multiple_proc_multiple_row_ufunction'"
  detail = 'multiple rows in the sampler using the U learning function;'
[]

[statistics]
  type = JSONDiff
  input = statistics_vpp.i
  jsondiff = statistics_vpp_out.json

  detail = "vectors of data from the postprocessing system and include"
[]

[ci]
  type = JSONDiff
  input = statistics_vpp_ci.i
  jsondiff = statistics_vpp_ci_out.json

  detail = "confidence level intervals for statistics calculations."
[]

[statistics]
  type = JSONDiff
  input = statistics_reporter.i
  jsondiff = statistics_reporter_out.json

  detail = "vector of data from the reporting system; including"
[]

[ci]
  type = JSONDiff
  input = statistics_reporter_ci.i
  jsondiff = statistics_reporter_ci_out.json

  detail = "confidence level intervals for statistics calculations"
[]

[error]
  type = RunException
  input = statistics_reporter_error.i
  expect_err = "The following reporter value\(s\) do not have a type supported by the StatisticsReporter:\n\s+const/const"

  detail = "and error if the supplied type is not supported."
[]

[basic]
  type = JSONDiff
  input = bca.i
  jsondiff = bca_out.json

  detail = "of a vector of data"
[]

[vector]
  type = JSONDiff
  input = bca_main.i
  jsondiff = bca_main_out.json

  detail = "of a vector of vector data"
  valgrind = HEAVY
[]

[replicated]
  type = JSONDiff
  input = bca_distributed.i
  jsondiff = bca_distributed_out.json
  cli_args = "VectorPostprocessors/data/parallel_type=replicated"
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is replicated and"
[]

[distributed]
  type = JSONDiff
  input = bca_distributed.i
  jsondiff = bca_distributed_out.json
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is distributed."
[]

[no_ci_levels]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels=''"
  expect_err = "If the 'ci_method' parameter is supplied then the 'ci_levels' must also be supplied with values in \(0, 1\)"

  detail = "the confidence level intervals are omitted;"
[]

[ci_levels_zero]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels='0'"
  expect_err = "The supplied levels must be greater than zero"

  detail = "the confidence level intervals are less than or equal to zero;"
[]

[ci_levels_one]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels='1'"
  expect_err = "The supplied levels must be less than 1\.0"

  detail = "the confidence level intervals are greater than or equal to one;"
[]

[no_input]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/vectorpostprocessors=''"
  expect_err = "The 'vectorpostprocessors' and/or 'reporters' parameters must be defined and non-empty"

  detail = "input is not provided."
[]

[basic]
  type = JSONDiff
  input = percentile.i
  jsondiff = percentile_out.json

  detail = "of a vector of data"
[]

[vector]
  type = JSONDiff
  input = percentile_main.i
  jsondiff = percentile_main_out.json

  detail = "of a vector of vector data"
  valgrind = HEAVY
[]

[replicated]
  type = JSONDiff
  input = percentile_distributed.i
  jsondiff = percentile_distributed_out.json
  cli_args = "VectorPostprocessors/data/parallel_type=replicated"
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is replicated or"
[]

[distributed]
  type = JSONDiff
  input = percentile_distributed.i
  jsondiff = percentile_distributed_out.json
  prereq = percentile/replicated
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "distributed across processors."
[]

[distributed]
  type = JSONDiff
  input = dp_main.i
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on distributed data sets;'
[]

[root]
  type = JSONDiff
  input = dp_main.i
  cli_args = 'Reporters/storage/parallel_type=ROOT'
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on root processor;'
[]

[distributed]
  type = JSONDiff
  input = dp_main.i
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on distributed data sets;'
  cli_args = 'Samplers/directperturbation/perturbation_method=forward_difference'
[]

[root]
  type = JSONDiff
  input = dp_main.i
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on root processor;'
  cli_args = 'Reporters/storage/parallel_type=ROOT Samplers/directperturbation/perturbation_method=forward_difference'
[]

[relative_sensitivity]
  type = JSONDiff
  input = dp_main.i
  cli_args = 'Reporters/sensitivities/relative_sensitivity=true Outputs/file_base=relative_sens'
  jsondiff = relative_sens.json
  allow_test_objects = true
  requirement = 'The system shall support the ability to compute direct perturbation-based relative sensitivity values.'
[]

[from_parallelstorage]
  type = JSONDiff
  requirement = 'The system should be able to map the solutions in a snapshot container into a latent space.'
  input = map_main.i
  jsondiff = 'map_training_data.json'
  min_parallel = 2
  max_parallel = 2
  petsc_version = ">=3.14"
[]

[from_solutions]
  # Note: this test should result in the same reporter values as the one above. Due to reporter naming difference, we
  # needed two different gold files. If something causes mismatch between the values in the two gold files, look for bugs.
  type = JSONDiff
  requirement = 'The system should be able to extract and map variables in nonlinear system into a latent (low-dimensional) space.'
  input = load_main.i
  cli_args = "worker:VariableMappings/inactive='' worker:Reporters/inactive=''"
  jsondiff = 'map_variable.json'
  min_parallel = 2
  max_parallel = 2
  prereq = from_parallelstorage
  petsc_version = ">=3.14"
[]

[distributed]
  type = JSONDiff
  input = morris.i
  jsondiff = morris_out.json
  allow_test_objects = true
  detail = 'distributed data;'
[]

[replicated]
  type = JSONDiff
  input = morris.i
  cli_args = 'VectorPostprocessors/gfun/parallel_type=REPLICATED'
  jsondiff = morris_out.json
  allow_test_objects = true
  detail = 'replicated data;'
[]

[distributed]
  type = JSONDiff
  input = morris_main.i
  jsondiff = morris_main_out.json
  allow_test_objects = true
  method = 'opt'
  detail = 'distributed;'
[]

[root]
  type = JSONDiff
  input = morris_main.i
  cli_args = 'Reporters/storage/parallel_type=ROOT'
  jsondiff = morris_main_out.json
  allow_test_objects = true
  method = 'opt'
  detail = 'on root processor;'
[]

[print-p2]
  type = JSONDiff
  requirement = 'The system should be able to print serialized snapshots in a distributed fashion in a json format.'
  input = parallel_storage_main.i
  jsondiff = 'parallel_storage_main_out.json parallel_storage_main_out.json.1'

  min_parallel = 2
  max_parallel = 2
[]

[sobol]
  type = JSONDiff
  input = sobol.i
  jsondiff = sobol_out.json
  allow_test_objects = true

  requirement = "The system shall support the ability to compute first, second, and total-effect Sobol sensitivity indices with a reporter."
[]

[sobol_vec]
  type = JSONDiff
  input = sobol_main.i
  jsondiff = sobol_main_out.json
  allow_test_objects = true

  requirement = "The system shall support the ability to compute Sobol sensitivity indices for vector-type data."
  method = 'opt'
[]

[replicated]
  type = 'JSONDiff'
  input = 'statistics.i'
  jsondiff = 'statistics_out.json'
  allow_test_objects = true

  detail = 'is replicated and'
[]

[distributed]
  type = 'JSONDiff'
  input = 'statistics_distributed.i'
  jsondiff = 'statistics_distributed_out.json'
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = 'distributed.'
[]

[reporter]
  type = JSONDiff
  input = statistics_main.i
  jsondiff = statistics_main_out.json
  requirement = 'The system shall be capable of computing the statistics from vector reporter values.'
[]

[serial]
  type = JSONDiff
  input = sample_only.i
  jsondiff = sample_only_out.json
  detail = 'serialized output;'
[]

[parallel]
  type = JSONDiff
  input = sample_only.i
  cli_args = 'Reporters/matrix/parallel_type=distributed Outputs/file_base=sample_only_dist'
  min_parallel = 3
  max_parallel = 3
  jsondiff = 'sample_only_dist.json sample_only_dist.json.1 sample_only_dist.json.2'
  detail = 'distributed output;'
[]

[names]
  type = JSONDiff
  input = sample_only.i
  cli_args = 'Reporters/matrix/sampler_column_names=\'foo bar fah lala\' Outputs/file_base=sample_only_names'
  jsondiff = sample_only_names.json
  detail = 'specified column names;'
[]

[simulation]
  type = CSVDiff
  input = stochastic_matrix.i
  csvdiff = stochastic_matrix_out_matrix_0001.csv
  requirement = 'The system shall be able to output samples from a sampler along side stochastic simulation results.'
[]

[wrong_num_names]
  type = RunException
  input = sample_only.i
  cli_args = 'Reporters/matrix/sampler_column_names=foobar'
  expect_err = 'The number of column names specified \(1\) does not match the number of sampler columns \(4\)\.'
  detail = 'inputting the wrong number of column names;'
[]

[wrong_sampler]
  type = RunException
  input = stochastic_matrix.i
  cli_args = 'Samplers/wrong/type=InputMatrix Samplers/wrong/matrix=\'1 2 3 4\' Reporters/matrix/sampler=wrong'
  expect_err = 'Attempting to create a stochastic vector with a different sampler \(sample\) than the one specified at input \(wrong\)\.'
  detail = 'using a different sampler than the transfer\'s;'
[]

[stats]
  type = JSONDiff
  input = stats.i
  jsondiff = stats_out.json

  detail = "compute statistics."
[]

[ais]
  requirement = 'The system shall include an Adaptive Importance Sampling method for sampling distribution data.'
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'ais.i'
  jsondiff = 'ais_out.json'
  valgrind = none
[]

[ais_error1]
  type = RunException
  input = 'ais_error1.i'
  detail = 'the selected sampler type is not of an adaptive type when performing adaptive decision;'
  expect_err = "The selected sampler is not an adaptive sampler."
[]

[pss1]
  requirement = 'The system shall include an Parallel Subset Simulation method for sampling distribution data.'
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'pss.i'
  jsondiff = 'pss_out.json'
  valgrind = none
[]

[pss_error1]
  type = RunException
  input = 'pss_error1.i'
  detail = 'the selected sampler type is not of an adaptive type. '
  expect_err = "The selected sampler is not an adaptive sampler."
[]

[test]
  type = CSVDiff
  input = grid.i
  csvdiff = grid_out_data_0000.csv

  requirement = "The system shall include the ability to create a 1D Cartesian sampling scheme."
[]

[no_triplet]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/linear_space_items='1 1'"
  expect_err = "The number of numeric items must be divisible by 3; min, max, divisions for each item are required."
  match_literal = True
  detail = 'the specified grid does not form a triplet,'
[]

[wrong_num_values]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/nominal_values='1 1'"
  expect_err = "The number of values specified must match the number of triplets in 'linear_space_items'."
  match_literal = True
  detail = 'the number of nominal values specified does not match the number of sample columns,'
[]

[non_int]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/linear_space_items='1 1 0.1 1 1 1 1 1 1'"
  expect_err = "The third entry for each item must be an integer; it provides the number of entries in the resulting item vector."
  match_literal = True
  detail = 'the number of grid points is not an integer, or'
[]

[negative]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/linear_space_items='1 1 -1 1 1 1 1 1 1'"
  expect_err = "The third entry for each item must be positive; it provides the number of entries in the resulting item vector."
  match_literal = True
  detail = 'the number of grid points is negative.'
[]

[test]
  type = CSVDiff
  input = grid.i
  csvdiff = grid_out_data_0000.csv

  requirement = "The system shall include the ability to create a Cartesian product sampling scheme."
[]

[sampling_test]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_out_data_0000.csv csv_sampler_out_data_0001.csv'

  detail = 'distributed across processors, '
[]

[sampling_test_global]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_out_data_0000.csv csv_sampler_out_data_0001.csv'
  prereq = csv_sampler/sampling_test
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'replicated across processors, and '
[]

[sampling_test_distributed]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_distributed_out_data_0000.csv.0 csv_sampler_distributed_out_data_0000.csv.1 csv_sampler_distributed_out_data_0001.csv.0 csv_sampler_distributed_out_data_0001.csv.1'
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=csv_sampler_distributed_out'
  min_parallel = 2
  max_parallel = 2
  detail = 'distributed across processors with the output also distributed.'
[]

[csv_sampler_indices]
  requirement = "The CSV Sampler shall sample from a CSV file when column indices are provided."

  type = 'CSVDiff'
  input = 'csv_sampler_indices.i'
  csvdiff = 'csv_sampler_indices_out_data_0000.csv csv_sampler_indices_out_data_0001.csv'
[]

[csv_sampler_names]
  requirement = "The CSV Sampler shall sample from a CSV file when column names are provided."

  type = 'CSVDiff'
  input = 'csv_sampler_names.i'
  csvdiff = 'csv_sampler_indices_out_data_0000.csv csv_sampler_indices_out_data_0001.csv'
  cli_args = Outputs/file_base='csv_sampler_indices_out'
[]

[serial]
  type = CSVDiff
  input = directperturbation.i
  csvdiff = 'directperturbation_out_data_0001.csv'
  detail = "with replicated output;"
[]

[parallel]
  type = CSVDiff
  input = directperturbation.i
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=dp_cd_3'
  min_parallel = 3
  max_parallel = 3
  csvdiff = 'dp_cd_3_data_0001.csv.0 dp_cd_3_data_0001.csv.1 dp_cd_3_data_0001.csv.2'
  detail = "with parallel output;"
[]

[serial]
  type = CSVDiff
  input = directperturbation.i
  csvdiff = 'dp_fd_data_0001.csv'
  detail = "with replicated output;"
  cli_args = 'Outputs/file_base=dp_fd Samplers/directperturbation/perturbation_method=forward_difference'
[]

[parallel]
  type = CSVDiff
  input = directperturbation.i
  cli_args = 'Samplers/directperturbation/perturbation_method=forward_difference VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=dp_fd_3'
  min_parallel = 3
  max_parallel = 3
  csvdiff = 'dp_fd_3_data_0001.csv.0 dp_fd_3_data_0001.csv.1 dp_fd_3_data_0001.csv.2'
  detail = "with parallel output;"
[]

[constant]
  type = JSONDiff
  input = main.i
  jsondiff = main_constant_size_out.json
  allow_test_objects = true
  cli_args = "Samplers/dynamic/increment_rows=0 Outputs/file_base=main_constant_size_out"

  detail = "that remain constant in size and"
[]

[dynamic]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  allow_test_objects = true

  detail = "that are dynamic in size."
[]

[initial]
  type = CSVDiff
  input = initial.i
  csvdiff = 'initial_out_data_0000.csv initial_out_data_0001.csv initial_out_data_0002.csv initial_out_data_0003.csv'

  requirement = 'The system shall support the creation of data sampled from distribution during the initial setup of a simulation.'
[]

[sampling_test]
  type = CSVDiff
  input = input_matrix.i
  csvdiff = input_matrix_out_data_0001.csv
  requirement = 'The system shall be able to sample an input-defined matrix.'
[]

[global]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "using global matrix,"
[]

[local]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "a local matrix,"
[]

[row]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "or row-by-row."
[]

[global]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "using global matrix,"
[]

[local]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "a local matrix,"
[]

[row]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "or row-by-row."
[]

[visualize]
  type = CheckFiles
  command = "./latin_hypercube.py"
  check_files = latin_hypercube.pdf
  required_python_packages = 'pandas numpy matplotlib'
  valgrind = NONE

  requirement = "The system shall include a utility that visually displays results of plotting Latin Hypercube test."
[]

[pmcmc_base]
  type = 'JSONDiff'
  input = main_base.i
  allow_test_objects = true
  jsondiff = 'mcmc_base.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the base MCMC class; '
[]

[pmcmc_base_bounds]
  type = 'JSONDiff'
  input = main_base.i
  allow_test_objects = true
  jsondiff = 'mcmc_base_bounds.json'
  cli_args = "Samplers/sample/lower_bound='-2.0 -2.0' Samplers/sample/upper_bound='2.0 2.0' Outputs/file_base='mcmc_base_bounds'"
  min_parallel = 4
  max_parallel = 4
  detail = 'using the base MCMC class with bounds specified; '
[]

[imh_5prop]
  type = 'JSONDiff'
  input = main_imh.i
  allow_test_objects = true
  jsondiff = 'imh_5prop.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the independent Metropolis-Hastings class; '
[]

[des_5prop]
  type = 'JSONDiff'
  input = main_des.i
  allow_test_objects = true
  jsondiff = 'des_5prop.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the differential evolution sampler class; '
[]

[ss_5prop]
  type = 'JSONDiff'
  input = main_ss.i
  allow_test_objects = true
  jsondiff = 'ss_5prop.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the stretch sampler class; '
[]

[des_5prop_var]
  type = 'JSONDiff'
  input = main_des_var.i
  allow_test_objects = true
  jsondiff = 'des_5prop_var.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the differential evolution sampler class with variance inferred; '
[]

[pmcmc_base_bounds1]
  type = RunException
  input = main_base.i
  cli_args = "Samplers/sample/lower_bound='-2.0 -2.0'"
  detail = 'only the lower bounds specified; '
  expect_err = "Both lower and upper bounds should be specified."
[]

[pmcmc_base_bounds2]
  type = RunException
  input = main_base.i
  cli_args = "Samplers/sample/lower_bound='-2.0 -2.0' Samplers/sample/upper_bound='2.0'"
  detail = 'the specified bounds are not of the same size; '
  expect_err = "Lower and upper bounds should be of the same size."
[]

[pmcmc_base_sizes]
  type = RunException
  input = main_base.i
  cli_args = "Samplers/sample/initial_values='0.1'"
  detail = 'the priors, bounds, and initial values are not of the same size; '
  expect_err = "The priors and initial values should be of the same size."
[]

[imh_std_sizes]
  type = RunException
  input = main_imh.i
  cli_args = "Samplers/sample/std_prop='0.05'"
  detail = 'the proposal stds, initial values, and priors are not of the same size; '
  expect_err = "The number of proposal stds, initial values, and priors should be the same."
[]

[des_scale_sizes]
  type = RunException
  input = main_des.i
  cli_args = "Samplers/sample/scales='1.0'"
  detail = 'the number of scales provided do not match the number of tunable params; '
  expect_err = "The number of scales provided should match the number of tunable params."
[]

[uniform]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_out_data_0000.csv monte_carlo_uniform_out_data_0001.csv'

  detail = 'a uniform distribution distributed across processors,'
[]

[uniform_global]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_out_data_0000.csv monte_carlo_uniform_out_data_0001.csv'
  prereq = monte_carlo/uniform
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'a uniform distribution replicated across processors,'
[]

[uniform_distributed]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_distributed_out_data_0000.csv.0 monte_carlo_uniform_distributed_out_data_0000.csv.1 monte_carlo_uniform_distributed_out_data_0001.csv.0 monte_carlo_uniform_distributed_out_data_0001.csv.1'
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=monte_carlo_uniform_distributed_out'
  min_parallel = 2
  max_parallel = 2
  detail = 'a uniform distribution distributed across processors (output is also distributed),'
[]

[weibull]
  type = 'CSVDiff'
  input = 'monte_carlo_weibull.i'
  csvdiff = 'monte_carlo_weibull_out_data_0000.csv monte_carlo_weibull_out_data_0001.csv'

  detail = 'a Wiebull distribution distributed across processors, and'
[]

[weibull_global]
  type = 'CSVDiff'
  input = 'monte_carlo_weibull.i'
  csvdiff = 'monte_carlo_weibull_out_data_0000.csv monte_carlo_weibull_out_data_0001.csv'
  prereq = monte_carlo/weibull
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'a Weibull distribution replicated across processors.'
[]

[serial]
  type = CSVDiff
  input = morris.i
  csvdiff = 'morris_out_data_0000.csv morris_out_data_0001.csv'
  detail = "with replicated output;"
[]

[parallel]
  type = CSVDiff
  input = morris.i
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=morris3'
  min_parallel = 3
  max_parallel = 3
  csvdiff = 'morris3_data_0000.csv.0 morris3_data_0000.csv.1 morris3_data_0000.csv.2 morris3_data_0001.csv.0 morris3_data_0001.csv.1 morris3_data_0001.csv.2'
  detail = "with parallel output;"
[]

[parallel_more]
  type = CSVDiff
  input = morris.i
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=morris5'
  min_parallel = 5
  max_parallel = 5
  csvdiff = 'morris5_data_0000.csv.0 morris5_data_0000.csv.1 morris5_data_0000.csv.2 morris5_data_0000.csv.3 morris5_data_0000.csv.4
                 morris5_data_0001.csv.0 morris5_data_0001.csv.1 morris5_data_0001.csv.2 morris5_data_0001.csv.3 morris5_data_0001.csv.4'
  detail = "with more processors than trajectories;"
[]

[serial]
  type = CSVDiff
  input = 'nested_monte_carlo.i'
  csvdiff = 'nested_monte_carlo_out_data_0001.csv'
  max_parallel = 1
  detail = 'in serial;'
[]

[parallel]
  type = CSVDiff
  input = 'nested_monte_carlo.i'
  csvdiff = 'nested_monte_carlo_out_data_0001.csv'
  min_parallel = 3
  detail = 'in parallel;'
[]

[error]
  type = RunException
  input = nested_monte_carlo.i
  cli_args = "Samplers/nested_mc/num_rows='10 3'"
  expect_err = "There must be a set of distributions for each loop defined by 'num_rows'"
  requirement = 'The system shall report an error when the number of nested Monte Carlo loops does not match the number of sets of distributions.'
[]

[resample]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_data_0000.csv'

  detail = "with the re-sampling matrix and"
[]

[no_resample]
  type = CSVDiff
  input = sobol.i
  cli_args = 'Samplers/sobol/resample=false Outputs/file_base=sobol_no_resample_out'
  csvdiff = 'sobol_no_resample_out_data_0000.csv'

  detail = "without the re-sampling matrix."
[]

[row_mismatch]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sample/num_rows=42"
  expect_err = "The supplied Sampler objects must have the same number of rows"

  detail = "with differing number of rows;"
[]

[col_mismatch]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sample/distributions='d0 d1'"
  expect_err = "The supplied Sampler objects must have the same number of columns"

  detail = "with differing number of columns; and"
[]

[same_matrix]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sobol/sampler_a=resample"
  expect_err = "The supplied sampler matrices must not be the same"

  detail = "if the matrices are the same."
[]

[VectorPostprocessors_sampler]
  requirement = 'The system shall be able to perform stochastic sampling based on global vector data.'
  type = 'CSVDiff'
  input = 'vectorpostprocessor.i'
  csvdiff = 'vectorpostprocessor_out_data_0002.csv'
[]

[cv_real]
  type = JSONDiff
  input = train_and_cv.i
  allow_test_objects = true
  cli_args = "Trainers/train/cv_surrogate='surrogate'"
  jsondiff = 'train_and_cv_out.json'
  detail = 'for a 1-dimensional polynomial regression surrogate with Real'
[]

[cv_vector_real]
  type = JSONDiff
  input = poly_reg_vec.i
  jsondiff = 'poly_reg_vec_out.json'
  detail = ' and std::vector<Real> response types.'
[]

[surr-error]
  type = RunException
  input = train_and_cv.i
  allow_test_objects = true
  expect_err = "To perform cross-validation, the option cv_surrogate needs to be specified to provide a Surrogate object for training and evaluation."
  detail = ' and throw an error if a SurrogateModel is not provided.'
[]

[GP_training]
  type = CheckFiles
  input = GP_squared_exponential_training.i
  check_files = 'gauss_process_training_GP_avg_trainer.rd/data'
  allow_test_objects = true
  detail = 'training a Gaussian process model and '
[]

[GP_testing]
  type = CSVDiff
  input = GP_squared_exponential_testing.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_testing_out_samp_avg_0002.csv'
  detail = 'evaluating the trained Gaussian process model'
  prereq = store_load/GP_training
[]

[GP_squared_exponential]
  type = CSVDiff
  input = GP_squared_exponential.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_out_samp_avg_0002.csv GP_squared_exponential_out_results_0002.csv'
  detail = 'a squared exponential kernel;'
[]

[GP_exponential]
  type = CSVDiff
  input = GP_exponential.i
  allow_test_objects = true
  csvdiff = 'GP_exponential_out_samp_avg_0002.csv GP_exponential_out_results_0002.csv'
  detail = 'an exponential kernel;'
[]

[GP_Matern_half_int]
  type = CSVDiff
  input = GP_Matern_half_int.i
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_out_samp_avg_0002.csv GP_Matern_half_int_out_results_0002.csv'
  detail = 'Matern half integer kernel;'
[]

[GP_squared_exponential_tuned_adam]
  type = CSVDiff
  input = GP_squared_exponential_tuned_adam.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_tuned_adam_out_samp_avg_0002.csv GP_squared_exponential_tuned_adam_out_results_0002.csv GP_squared_exponential_tuned_adam_out_hyperparams_0002.csv'
  detail = 'a squared exponential kernel using Adam;'
[]

[GP_exponential_tuned_adam]
  type = CSVDiff
  input = GP_exponential_tuned_adam.i
  allow_test_objects = true
  csvdiff = 'GP_exponential_tuned_adam_out_samp_avg_0002.csv GP_exponential_tuned_adam_out_results_0002.csv GP_exponential_tuned_adam_out_hyperparams_0002.csv'
  detail = 'an exponential kernel using Adam;'
[]

[GP_Matern_half_int_tuned_adam]
  type = CSVDiff
  input = GP_Matern_half_int_tuned_adam.i
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_tuned_adam_samp_avg_0002.csv GP_Matern_half_int_tuned_adam_results_0002.csv GP_Matern_half_int_tuned_adam_hyperparams_0002.csv'
  detail = 'a Matern half integer kernel using Adam;'
[]

[GP_Matern_half_int_tuned_adam_minibatch]
  type = CSVDiff
  input = GP_Matern_half_int_tuned_adam.i
  cli_args = "Trainers/GP_avg_trainer/batch_size=10 Outputs/out/file_base='GP_Matern_half_int_tuned_adam_minibatch'"
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_tuned_adam_minibatch_samp_avg_0002.csv GP_Matern_half_int_tuned_adam_minibatch_results_0002.csv GP_Matern_half_int_tuned_adam_minibatch_hyperparams_0002.csv'
  detail = 'a Matern half integer kernel using Adam with mini-batch sampling;'
[]

[optimization_adam_batch_size]
  type = RunException
  input = GP_squared_exponential_tuned_adam.i
  cli_args = "Trainers/GP_avg_trainer/batch_size='100'"
  detail = 'the batch size is greater than the training data set size for Adam optimization. '
  expect_err = "Batch size cannot be greater than the training data set size."
[]

[train_and_evaluate]
  requirement = 'The system shall be able to train and evaluate a libtorch-based neural network in the same input file.'
  type = CSVDiff
  input = train_and_evaluate.i
  allow_test_objects = true
  cli_args = "Trainers/train/read_from_file=false"
  csvdiff = 'train_and_evaluate_out_results_0002.csv'
  libtorch = true
[]

[train]
  requirement = 'The system shall be able to train a libtorch-based neural network.'
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data mynet.pt'
  cli_args = "Trainers/train/read_from_file=false"
  allow_test_objects = true
  libtorch = true
[]

[evaluate]
  requirement = 'The system shall be able to evaluate a previously trained, libtorch-based neural network.'
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = train
  libtorch = true
[]

[retrain]
  requirement = 'The system shall be able to retrain a pretrained and saved libtorch-based neural network.'
  type = CSVDiff
  input = retrain.i
  allow_test_objects = true
  cli_args = "Trainers/train/read_from_file=true"
  csvdiff = 'retrain_out_results_0002.csv'
  prereq = evaluate
  libtorch = true
[]

[train-with-rel-tol]
  requirement = 'The system shall be able to train a libtorch-based neural network using a relative tolerance instead of fixed epoch number.'
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data mynet.pt'
  cli_args = "Trainers/train/read_from_file=false Trainers/train/rel_loss_tol=0.1"
  allow_test_objects = true
  libtorch = true
  prereq = retrain
[]

[train-standardized]
  requirement = 'The system shall be able to train a libtorch-based neural network with standardized input and output parameters.'
  type = CheckFiles
  input = train.i
  check_files = 'standardized_train.rd/data mynet.pt'
  cli_args = "Trainers/train/read_from_file=false Trainers/train/standardize_input=true Trainers/train/standardize_output=true Outputs/file_base=standardized"
  allow_test_objects = true
  libtorch = true
[]

[evaluate-standardized]
  requirement = 'The system shall be able to evaluate a previously trained, libtorch-based neural network with standardized input and output parameters.'
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_standardized_results_0002.csv'
  cli_args = "Surrogates/surr/filename='standardized_train.rd' Outputs/file_base=evaluate_standardized"
  prereq = train-standardized
  libtorch = true
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_poly_chaos.rd/data'
  allow_test_objects = true

  detail = "trained with replicated stochastic data and"
[]

[evaluate]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train # Creates data files that this test uses
  allow_warnings = true
  detail = "evaluated separately (with replicated data);"
[]

[train_distributed]
  type = CheckFiles
  input = train.i
  cli_args = Reporters/storage/parallel_type=DISTRIBUTED
  check_files = 'train_out_poly_chaos.rd/data'
  prereq = surrogate/evaluate
  allow_test_objects = true

  detail = "trained with distributed stochastic data and"
[]

[evaluate_distributed]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train_distributed # Creates data files that this test uses
  allow_warnings = true
  detail = "evaluated separately (with distributed data);"
[]

[train_three_ranks]
  type = CheckFiles
  input = train.i
  cli_args = Reporters/storage/parallel_type=DISTRIBUTED
  check_files = 'train_out_poly_chaos.rd/data'
  prereq = surrogate/evaluate_distributed
  allow_test_objects = true
  min_parallel = 3

  detail = "trained with distributed stochastic data;"
[]

[evaluate_two_ranks]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train_three_ranks # Creates data files that this test uses
  max_parallel = 2

  detail = "evaluated separately with a different number of processors;"
[]

[evaluate_and_train]
  type = JSONDiff
  input = train_and_evaluate.i
  cli_args = 'Outputs/file_base=evaluate_out'
  jsondiff = evaluate_out.json
  allow_test_objects = true

  detail = "and be trained and evaluated in memory with a single input file."
[]

[untuned]
  type = 'JSONDiff'
  input = mogp_lmc.i
  jsondiff = "mogp_lmc_out.json"
  detail = "without tuning the hyperparameters."
[]

[tuned]
  type = 'JSONDiff'
  input = mogp_lmc_tuned.i
  jsondiff = "mogp_lmc_tuned_out.json"
  detail = "with tuning the hyperparameters."
  abs_zero = 1e-6
[]

[load]
  type = 'JSONDiff'
  input = mogp_lmc_load.i
  jsondiff = "mogp_lmc_load_out.json"
  detail = "and load the trained surrogate from a file."
  prereq = lmc/tuned
  skip_keys = "reporters train_params"
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data'
  allow_test_objects = true
  detail = 'training then '
[]

[load]
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = nearest_point/train # Creates data files that this test uses
  detail = 'evaluating, '
[]

[train_and_load]
  type = CSVDiff
  input = cartesian.i
  allow_test_objects = true
  csvdiff = 'cartesian_out_results_0002.csv'
  detail = 'training and loading, and '
[]

[predictors]
  type = CSVDiff
  input = predictor_response.i
  allow_test_objects = true
  cli_args = 'Outputs/file_base=evaluate_out'
  csvdiff = 'evaluate_out_results_0002.csv'
  detail = 'using explictly specified predictors.'
[]

[vector]
  type = JSONDiff
  input = np_vec.i
  jsondiff = 'np_vec_out.json'
  requirement = 'The system shall be able to create nearest point surrogates with vector-type response.'
[]

[train]
  type = CheckFiles
  input = trainer.i
  check_files = 'trainer_out_pod_rb.rd/data'
  allow_test_objects = true
  detail = 'training using known 4D data '
  max_parallel = 5 # We need fewer processors than samples
[]

[load]
  type = CSVDiff
  input = surr.i
  allow_test_objects = true
  csvdiff = 'surr_out_res_0001.csv'
  prereq = pod_boundary/train # Creates data files that this test uses
  detail = 'and then evaluating new samples separately for new data.'
[]

[trainer_name]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'MultiApps/sub/trainer_name="doesnt_exist"'
  detail = 'the trainer object cannot be found.'
  expect_err = "Unable to find user object with name 'doesnt_exist'"
  max_parallel = 3
[]

[max_procs_per_app]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'MultiApps/sub/max_procs_per_app=2 Samplers/sample/max_procs_per_row=2'
  detail = 'the trainer is run with more than one processors per subapp.'
  expect_err = "does not support more than one processors per subapp"
[]

[more_procs_than_samples]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  detail = 'the number of samples is lower than the number of processors.'
  expect_err = "PODFullSolveMultiApp needs to be run with fewer processors \(detected (\d+)\) than samples \(detected 3\)"
  min_parallel = 4
[]

[trainer_name]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Transfers/snapshot/trainer_name="doesnt_exist"'
  detail = 'the trainer object cannot be found.'
  expect_err = "Unable to find user object with name 'doesnt_exist'"
  max_parallel = 3
[]

[var_names]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = "Trainers/pod_rb/var_names='doesnt_exist'"
  detail = 'the variable names cannot be found on sub-applications, '
  expect_err = "Variable 'doesnt_exist' not found on sub-application [0-9]+!"
  max_parallel = 3
[]

[en_limits]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = "Trainers/pod_rb/error_res='1e-2 1e-3'"
  detail = 'the number of energy limits and variable names do not match, '
  expect_err = "The number of elements is not equal to the number of elements in 'var_names'!"
  max_parallel = 3
[]

[tag_types]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/tag_types="op"'
  detail = 'the number of tag names and tag types do not match, '
  expect_err = "The number of elements is not equal to the number of elements in 'tag_names'!"
  max_parallel = 3
[]

[dir_tag_names]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/tag_types="doesnt_exist doesnt_exist doesnt_exist"'
  detail = "the Dirichlet tag types do not exist, "
  expect_err = "Tag type 'doesnt_exist' is not valid, available names are: op, op_dir, src, src_dir"
  max_parallel = 3
[]

[wrong_res_transfer]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/execute_on="final"'
  detail = 'and the residual generation is called before having the basis vectors.'
  expect_err = "There are no basis vectors available for residual generation. This indicates that the bases have not been created yet. The most common cause of this error is the wrong setting of the 'execute_on' flags in the PODFullSolveMultiApp and/or PODReducedBasisTrainer."
  match_literal = true
  max_parallel = 3
[]

[var_name]
  type = RunException
  input = trainer_and_surr.i
  allow_test_objects = true
  cli_args = "Surrogates/rbpod/change_rank='doesnt_exist'"
  detail = "the number of inputs in 'change_rank' and 'new_ranks' is not the same."
  expect_err = "The size of 'new_ranks' is not equal to the size of 'change_rank' 0 != 1"
[]

[train]
  type = CheckFiles
  input = trainer.i
  check_files = 'trainer_out_pod_rb.rd/data'
  allow_test_objects = true
  detail = 'training using known 3D data, '
  max_parallel = 3 # We need fewer processors than samples
[]

[save_ev]
  type = CSVDiff
  input = trainer.i
  csvdiff = 'eigenvalues_u.csv'
  cli_args = "Trainers/pod_rb/filenames='eigenvalues_u.csv'"
  allow_test_objects = true
  detail = 'saving the eigenvalues, '
  max_parallel = 3 # We need fewer processors than samples
[]

[load]
  type = CSVDiff
  input = surr.i
  allow_test_objects = true
  csvdiff = 'surr_out_res_0001.csv'
  prereq = pod_internal/train # Creates data files that this test uses
  detail = 'then evaluating new samples separately for new data, '
[]

[train_and_load]
  type = CSVDiff
  input = trainer_and_surr.i
  allow_test_objects = true
  csvdiff = 'trainer_and_surr_out_res_0002.csv'
  detail = 'and doing both together in one input file.'
  max_parallel = 3 # We need fewer processors than samples
[]

[monte_carlo_integration]
  type = CSVDiff
  input = main_2d_mc.i
  allow_test_objects = true
  csvdiff = 'main_2d_mc_out_pc_samp_0002.csv'
  detail = 'MonteCarlo sampler with Uniform distribution, '
[]

[monte_carlo_ols]
  type = JSONDiff
  input = ols_test.i
  jsondiff = ols_test_out.json
  detail = 'MonteCarlo sampler with Uniform distribution using least-squares regression, '
[]

[monte_carlo_ridge]
  type = JSONDiff
  input = ols_test.i
  cli_args = 'Trainers/train_ols/penalty=1.0 Trainers/train_pet_ols/penalty=1.0 Outputs/file_base=ridge_test_out'
  jsondiff = ridge_test_out.json
  detail = 'MonteCarlo sampler with Uniform distribution using Ridge regression, '
[]

[gauss_legendre_integration]
  type = CSVDiff
  input = main_2d_quad.i
  allow_test_objects = true
  csvdiff = 'main_2d_quad_out_pc_samp_0002.csv'
  detail = 'Quadrature sampler with Uniform distribution, and '
[]

[gauss_hermite]
  type = CSVDiff
  input = main_2dnorm_quad.i
  allow_test_objects = true
  csvdiff = 'main_2dnorm_quad_out_pc_samp_0002.csv'
  detail = 'Quadrature sampler with Normal distribution.'
[]

[legendre_moment_calculation]
  type = JSONDiff
  input = main_2d_quad_moment.i
  allow_test_objects = true
  jsondiff = 'main_2d_quad_moment_out.json'
  detail = 'statistical moments with Legendre polynomials, '
[]

[hermite_moment_calculation]
  type = JSONDiff
  input = main_2dnorm_quad_moment.i
  allow_test_objects = true
  jsondiff = 'main_2dnorm_quad_moment_out.json'
  detail = 'statistical moments with Hermite polynomials, '
[]

[legendre_local_sensitivity]
  type = JSONDiff
  input = main_2d_quad_locs.i
  allow_test_objects = true
  jsondiff = 'main_2d_quad_locs_out.json'
  detail = 'sampler and user defined local sensitivities with Legendre polynomials, '
[]

[hermite_local_sensitivity]
  type = JSONDiff
  input = main_2dnorm_quad_locs.i
  allow_test_objects = true
  jsondiff = 'main_2dnorm_quad_locs_out.json'
  detail = 'sampler and user defined local sensitivities with Hermite polynomials, and '
[]

[sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  jsondiff = 'sobol_out.json'
  detail = 'Sobol sensitivity indices.'
[]

[smolyak_sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  cli_args = 'Samplers/sample/sparse_grid=smolyak Outputs/file_base=smolyak'
  jsondiff = 'smolyak.json'
  detail = 'Smolyak and '
[]

[clenshaw_curtis_sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  cli_args = 'Samplers/sample/sparse_grid=clenshaw-curtis Outputs/file_base=clenshaw_curtis'
  jsondiff = 'clenshaw_curtis.json'
  detail = 'Clenshaw-Curtis methods.'
[]

[wrong_sampler]
  type = RunException
  input = error.i
  allow_test_objects = true
  expect_err = "Reporter value values/g_values of size [0-9]{3,4} does not match sampler size \([0-9]{2,3}\)."
  detail = "the number of samples does not match the number of results."
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data'
  allow_test_objects = true
  detail = 'training using known 3D data '
[]

[load]
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = poly_reg/train # Creates data files that this test uses
  detail = 'and then evaluating new samples separately for the same data'
[]

[train_and_load]
  type = CSVDiff
  input = train_and_evaluate.i
  allow_test_objects = true
  csvdiff = 'train_and_evaluate_out_results_0002.csv'
  detail = 'and then doing both on another 1D case.'
[]

[vector]
  type = JSONDiff
  input = poly_reg_vec.i
  jsondiff = 'poly_reg_vec_out.json'
  requirement = 'The system shall be able to create polynomial regression surrogate with vector-type response.'
[]

[n1]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal'
  max_parallel = 1

  detail = "on a single processor, "
[]

[n2]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal'
  prereq = normal/n1
  min_parallel = 2
  max_parallel = 2

  detail = "on multiple processors, "
[]

[n3]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_small_storage_0001.csv parent_small_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal
                  Samplers/mc/num_rows=3
                  Outputs/file_base=parent_small'
  min_parallel = 4
  max_parallel = 4

  detail = "and on more processors than samples."
[]

[n1]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  max_parallel = 1
  prereq = normal/n2

  detail = "on a single processor,"
[]

[n2]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = batch/n1

  detail = "on multiple processors, and"
[]

[n2_restore]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = batch/n2
  cli_args = MultiApps/runner/mode=batch-restore

  detail = "on multiple processors using in-memory sub-application restore."
[]

[batch_two_subapps]
  requirement = "The system shall include the ability to transfer stochastic results for two sub apps."
  issues = '#17079'
  type = CSVDiff
  input = parent_2sub.i
  csvdiff = 'parent_2sub_out_data0_0001.csv parent_2sub_out_data1_0001.csv parent_2sub_out_storage0_0001.csv parent_2sub_out_storage1_0001.csv'
[]

[StochasticToolsTransfer_execute_on_error]
  type = RunException
  input = parent.i
  cli_args = Transfers/data/execute_on=INITIAL
  expect_err = "The 'execute_on' parameter for the 'data' transfer was set"
  requirement = "The 'StochasticToolsTransfer object shall error if the 'execute_on' parameter is defined when the corresponding MultiApp object is running in batch mode."
[]

[StochasticToolsTransfer_execute_on_check]
  type = RunException
  input = parent.i
  cli_args = "Transfers/data/execute_on=INITIAL MultiApps/runner/mode=normal"
  expect_err = "MultiAppTransfer execute_on flags do not match associated from_multi_app execute_on flags"
  requirement = "The 'StochasticToolsTransfer' object shall error if the 'execute_on' parameter does not match the corresponding MultiApp object is running in normal mode."
[]

[direction_error]
  type = RunException
  input = parent.i
  cli_args = 'Transfers/data/to_multi_app=runner'
  expect_err = 'Transfers between multiapp are not currently supported for this transfer type'
  requirement = 'The system shall report a reasonable error if parameters for a trasnfer between multiapps are provided to stochastics transfer, which do not support this currently'
[]

[torch-parameters]
  type = JSONDiff
  input = libtorch_drl_control_trainer.i
  jsondiff = 'train_out.json train_out_runner0_json_out.json'
  skip_keys = 'reporters time time_step'
  requirement = "The system shall be able to transfer an artificial neural network between a trainer object and a control object residing on different apps."
  libtorch = true
[]

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_out_sub0.csv monte_carlo_out_sub1.csv monte_carlo_out_sub2.csv monte_carlo_out_sub3.csv monte_carlo_out_sub4.csv'
  requirement = "The system shall support the creation of a sub-application for each row of the stochastic data."
[]

[wrong_multi_app]
  type = RunException
  input = wrong_multi_app.i
  expect_err = "The 'multi_app' parameter must provide either a 'SamplerTransientMultiApp' or 'SamplerFullSolveMultiApp' object."
  requirement = "The system shall produce an error if neither a 'SamplerTransientMultiApp' nor `SamplerFullSolveMultiApp` is provided in SamplerPostprocessorTransfer."
[]

[require_stochastic_results]
  type = RunException
  input = require_stochastic_results.i
  expect_err = "The object prescribed by the 'to_vector_postprocessor' parameter in data must be a 'StochasticResults' object"
  requirement = "The system shall produce an error if the 'result' object in 'SamplerPostprocessorTransfer' is not a 'StochasticResults object'."
[]

[sobol_from_multiapp]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_storage_0002.csv parent_out_storage_0003.csv parent_out_storage_0004.csv parent_out_storage_0005.csv'

  requirement = "The system shall support the ability to transfer a single value from each sub-application for a set of stochastic data."
[]

[invalid_sub_pp_name]
  type = RunException
  input = parent.i
  cli_args = Transfers/data/from_postprocessor=wrong
  expect_err = "Unknown postprocesssor name 'wrong' on sub-application 'sub'"

  requirement = "The system shall report an error if the supplied name is invalid when attempting to
                  transfer a single value from a sub-application."
[]

[abort]
  type = RunException
  input = cartesian_diverge.i
  cli_args = 'MultiApps/sub/ignore_solve_not_converge=false
                  sub0:Executioner/error_on_dtmin=true'
  expect_err = "Solve failed and timestep already at or below dtmin, cannot continue!"

  detail = "abort run, "
[]

[last_computed]
  type = CSVDiff
  input = cartesian_diverge.i
  csvdiff = 'cartesian_diverge_out_storage_0001.csv cartesian_diverge_out_storage_0002.csv'

  detail = "transfer last computed postprocessor value, "
  valgrind = 'none' # too slow
[]

[nan]
  type = CheckFiles
  input = cartesian_diverge.i
  cli_args = 'Transfers/data/keep_solve_fail_value=false
                  Outputs/file_base=cartesian_diverge_nan'
  check_files = 'cartesian_diverge_nan_storage_0001.csv'
  file_expect_out = 'data:avg\nnan\n0\.269307.*\n0\.269307.*\n0\.269307.*\n0\.269307.*\n'
  detail = "or transfer NaN."
  valgrind = 'none' # too slow
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json

  detail = "in normal mode, "
[]

[batch]
  type = JSONDiff
  input = main_batch.i
  jsondiff = 'main_batch_out.json'

  detail = 'in batch mode, '
[]

[distributed]
  type = JSONDiff
  input = main.i
  cli_args = 'Reporters/storage/parallel_type=DISTRIBUTED Outputs/file_base=main_parallel'
  jsondiff = 'main_parallel.json main_parallel.json.1 main_parallel.json.2 main_parallel.json.3'
  min_parallel = 4
  max_parallel = 4

  detail = 'with distributed output, '
[]

[more_cpus]
  type = JSONDiff
  input = main_small.i
  jsondiff = 'main_small_out.json'
  min_parallel = 4

  detail = 'with more processors than samples, '
[]

[invalid_type]
  type = RunException
  input = main.i
  cli_args = "Transfers/data/from_reporter='mesh/sidesets'"
  expect_err = 'Reporter value mesh/sidesets is of unsupported type'

  detail = 'and error if transferring unsupported type.'
[]

[multiapp_type]
  type = RunException
  input = parent_multiapp_type_error.i
  expect_err = "The 'multi_app' parameter must provide either a 'SamplerTransientMultiApp' or 'SamplerFullSolveMultiApp' object."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if neither a 'SamplerTransientMultiApp' nor `SamplerFullSolveMultiApp` is provided in SamplerParameterTransfer."
[]

[control_missing]
  type = RunException
  input = parent_missing_control.i
  design = 'SamplerTransientMultiApp.md'
  expect_err = "The sub-application \(sub\) does not contain a SamplerReceiver control object."
  requirement = "The system shall produce an error if the sampler sub-application does not contain a Control object with the name 'stochastic'."
[]

[num_parameters_wrong]
  type = RunException
  input = parent_num_parameters_wrong.i
  expect_err = "Number of command line arguments does not match number of sampler columns."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if supplied vector of real values is not sized correctly within the SamplerParameterTransfer object."
[]

[not_vector]
  type = RunException
  input = parent_not_vector.i
  expect_err = "The Real parameter '[a-z,A-Z,/]+' expects a single value, but the vector supplying its values has a size of \d+\."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if a vector of values is supplied to a scalar parameter."
[]

[sampler_mismatch]
  type = RunException
  input = parent_transfer_wrong_sampler.i
  expect_err = "The supplied 'multi_app' must have the same Sampler object as this Transfer."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if sampling method differs between the sub-application and the associated sub-application data transfer."
[]

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_out_sub0.csv monte_carlo_out_sub1.csv monte_carlo_out_sub2.csv monte_carlo_out_sub3.csv monte_carlo_out_sub4.csv'

  detail = 'using a Monte Carlo and'
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_sub00.csv sobol_out_sub01.csv sobol_out_sub02.csv sobol_out_sub03.csv sobol_out_sub04.csv sobol_out_sub05.csv sobol_out_sub06.csv sobol_out_sub07.csv sobol_out_sub08.csv sobol_out_sub09.csv sobol_out_sub10.csv sobol_out_sub11.csv sobol_out_sub12.csv sobol_out_sub13.csv sobol_out_sub14.csv sobol_out_sub15.csv sobol_out_sub16.csv sobol_out_sub17.csv'

  detail = 'Sobol sampling scheme.'
[]

[monte_carlo]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_sub0.csv parent_out_sub1.csv parent_out_sub2.csv parent_out_sub3.csv parent_out_sub4.csv'
  requirement = "The system shall be capable of transferring vector data to sub-applications for each row of the stochastic data."
[]

[invalid_name]
  type = RunException
  input = parent.i
  cli_args = "Transfers/sub/parameters='foo bar fah lala' :Outputs/csv=false"
  expect_err = "The desired parameter '\w+' was not located for the 'stochastic' object, it either does not exist or has not been declared as controllable."
  detail = "if the vector parameter does not exist;"
[]

[extra_data]
  type = RunException
  input = parent.i
  cli_args = "Transfers/sub/parameters='BCs/left/value' :Outputs/csv=false"
  expect_err = "Number of command line arguments does not match number of sampler columns."
  detail = "if the sub-application does not consume all of the supplied data;"
[]

[batch_restore]
  detail = "batch-restore mode,"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'serialization_batch_restore.json serialization_batch_restore.json.1 serialization_batch_restore.json.2 serialization_batch_restore.json.3'
  cli_args = "Outputs/file_base=serialization_batch_restore Transfers/solution_transfer/serialize_on_root=true MultiApps/worker/mode=batch-restore"
  min_parallel = 4
  max_parallel = 4
[]

[batch_reset]
  detail = "batch-reset mode, and"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'serialization_batch_reset.json serialization_batch_reset.json.1 serialization_batch_reset.json.2 serialization_batch_reset.json.3'
  cli_args = "Outputs/file_base=serialization_batch_reset Transfers/solution_transfer/serialize_on_root=true MultiApps/worker/mode=batch-reset"
  min_parallel = 4
  max_parallel = 4
[]

[normal]
  detail = "normal mode."
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'serialization_normal.json serialization_normal.json.1 serialization_normal.json.2 serialization_normal.json.3'
  cli_args = "Outputs/file_base=serialization_normal Transfers/solution_transfer/serialize_on_root=true MultiApps/worker/mode=normal"
  min_parallel = 4
  max_parallel = 4
[]

[batch_restore]
  detail = "batch-restore mode,"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'no_serialization_batch_restore.json no_serialization_batch_restore.json.1 no_serialization_batch_restore.json.2 no_serialization_batch_restore.json.3'
  cli_args = "Outputs/file_base=no_serialization_batch_restore Transfers/solution_transfer/serialize_on_root=false MultiApps/worker/mode=batch-restore Transfers/solution_transfer/variables='u' Reporters/parallel_storage/variables='u'"
  min_parallel = 4
  max_parallel = 4
[]

[batch_reset]
  detail = "batch-reset mode, and"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'no_serialization_batch_reset.json no_serialization_batch_reset.json.1 no_serialization_batch_reset.json.2 no_serialization_batch_reset.json.3'
  cli_args = "Outputs/file_base=no_serialization_batch_reset Transfers/solution_transfer/serialize_on_root=false MultiApps/worker/mode=batch-reset Transfers/solution_transfer/variables='u' Reporters/parallel_storage/variables='u'"
  min_parallel = 4
  max_parallel = 4
[]

[normal]
  detail = "normal mode."
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'no_serialization_normal.json no_serialization_normal.json.1 no_serialization_normal.json.2 no_serialization_normal.json.3'
  cli_args = "Outputs/file_base=no_serialization_normal Transfers/solution_transfer/serialize_on_root=false MultiApps/worker/mode=normal Transfers/solution_transfer/variables='u' Reporters/parallel_storage/variables='u'"
  min_parallel = 4
  max_parallel = 4
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_sub00.csv sobol_out_sub01.csv sobol_out_sub02.csv sobol_out_sub03.csv sobol_out_sub04.csv sobol_out_sub05.csv sobol_out_sub06.csv sobol_out_sub07.csv sobol_out_sub08.csv sobol_out_sub09.csv sobol_out_sub10.csv sobol_out_sub11.csv sobol_out_sub12.csv sobol_out_sub13.csv sobol_out_sub14.csv sobol_out_sub15.csv sobol_out_sub16.csv sobol_out_sub17.csv'

  requirement = "The system shall support the creation of a sub-application for each row sampled data generated from a Sobol scheme."
[]

[create_surrogate]
  requirement = 'The system shall be able to create a surrogate model for the solution coordinates in the latent space.'
  type = CheckFiles
  input = create_mapping_main.i
  check_files = 'create_mapping_main_rom_polyreg_v.rd/data create_mapping_main_rom_polyreg_v_aux.rd/data create_mapping_main_mapping_pod_mapping.rd/data'
  min_parallel = 2
  max_parallel = 2
  petsc_version = ">=3.14"
[]

[with_surrogate]
  detail = 'coordinates determined by a surrogate model.'
  type = Exodiff
  input = inverse_map.i
  cli_args = "Outputs/file_base='with_surrogate'"
  exodiff = 'with_surrogate.e'
  min_parallel = 2
  max_parallel = 2
  prereq = create_surrogate
  petsc_version = ">=3.14"
[]

[with_manual_input]
  detail = 'manually specified coordinates.'
  type = Exodiff
  input = inverse_map.i
  cli_args = "Outputs/file_base='with_manual' UserObjects/im/surrogate=''"
  exodiff = 'with_manual.e'
  min_parallel = 2
  max_parallel = 2
  prereq = create_surrogate
  petsc_version = ">=3.14"
[]

[with_polyreg_csv]
  # Note: the reconstructed solution with 2 POD modes and a linear regression should be exact.
  detail = 'polynomial regression surrogates for the expansion coefficients.'
  type = CSVDiff
  input = inverse_map.i
  cli_args = "Outputs/file_base='with_surrogate'"
  csvdiff = 'with_surrogate.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = create_surrogate
  petsc_version = ">=3.14"
[]

[svd_1_proc_per_app]
  type = JSONDiff
  requirement = 'The system should be able to build a variable mapping by computing the SVD of a snapshot matrix in serial.'
  input = pod_mapping_main.i
  jsondiff = 'serial.json'
  cli_args = "Outputs/file_base=serial"
  skip_keys = "left_singular_vectors right_singular_vectors" # We do this because the negative of the eigenvectors are still eigenvectors and depending on the system it finds alternating signs
  max_parallel = 1
  petsc_version = ">=3.14"
[]

[svd_2_proc_per_app]
  type = JSONDiff
  requirement = 'The system should be able to build a variable mapping by computing the SVD of a snapshot matrix in parallel.'
  input = pod_mapping_main.i
  jsondiff = 'two_procs_per_app.json'
  skip_keys = "left_singular_vectors right_singular_vectors" # We do this because the negative of the eigenvectors are still eigenvectors and depending on the system it finds alternating signs
  cli_args = "Outputs/file_base=two_procs_per_app Transfers/solution_transfer/serialize_on_root=false"
  min_parallel = 2
  max_parallel = 2
  petsc_version = ">=3.14"
[]

[svd_2_proc_per_app_root]
  type = JSONDiff
  requirement = 'The system should be able to build a variable mapping by computing the SVD of a snapshot matrix in parallel with having data on only root processors.'
  input = pod_mapping_main.i
  jsondiff = 'two_procs_per_app_root.json'
  cli_args = "Outputs/file_base=two_procs_per_app_root Transfers/solution_transfer/serialize_on_root=true"
  skip_keys = "left_singular_vectors right_singular_vectors" # We do this because the negative of the eigenvectors are still eigenvectors and depending on the system it finds alternating signs
  min_parallel = 2
  max_parallel = 2
  petsc_version = ">=3.14"
[]

[multiple]
  type = CSVDiff
  input = parent.i
  csvdiff = parent_out_storage_0002.csv

  requirement = "The system shall support the collection of stochastic data from multiple sub-applications."
[]

[get_global_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples'
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'get global matrix;'
[]

[get_local_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples'
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'get local matrix;'
[]

[get_next_local_row]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row'
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'get next local row;'
[]

[get_local_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples  VectorPostprocessors/data/parallel_type=distributed'
  csvdiff = 'sampler_data_out_data_0001.csv.0 sampler_data_out_data_0001.csv.1 sampler_data_out_data_0001.csv.2'
  min_parallel = 3
  max_parallel = 3
  detail = 'get local matrix;'
[]

[get_next_local_row]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row  VectorPostprocessors/data/parallel_type=distributed'
  csvdiff = 'sampler_data_out_data_0001.csv.0 sampler_data_out_data_0001.csv.1 sampler_data_out_data_0001.csv.2'
  min_parallel = 3
  max_parallel = 3
  detail = 'get next local row;'
[]

[one]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist'"
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'one column;'
[]

[multi]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist dist dist' Outputs/file_base=multi_col"
  csvdiff = multi_col_data_0001.csv
  detail = 'multiple columns;'
[]

[lots]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist' Outputs/file_base=lots_col"
  csvdiff = lots_col_data_0001.csv
  detail = 'large number of columns;'
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = sobol_out_sobol_0002.csv
  allow_test_objects = true

  requirement = "The system shall support the ability to compute first, second, and total-effect Sobol sensitivity indices."
[]

[sobol_bootstrap]
  type = CSVDiff
  input = sobol_bootstrap.i
  csvdiff = sobol_bootstrap_out_sobol_0002.csv
  allow_test_objects = true

  requirement = "The system shall support the ability to compute confidence intervals on Sobol sensitivity indices."
[]

[replicated]
  type = CSVDiff
  input = parent.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=replicated_out'
  csvdiff = replicated_out_storage_0002.csv

  detail = "replicated on all processors and"
[]

[distributed]
  type = CSVDiff
  input = parent.i
  min_parallel = 3
  max_parallel = 3
  cli_args = 'VectorPostprocessors/storage/parallel_type=DISTRIBUTED Outputs/file_base=distributed_out'
  csvdiff = 'distributed_out_storage_0002.csv.0 distributed_out_storage_0002.csv.1 distributed_out_storage_0002.csv.2'

  detail = "distributed across many."
[]

[custom]
  type = CSVDiff
  input = parent.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=prefix_out Transfers/data/prefix=prefix'
  csvdiff = prefix_out_storage_0002.csv

  detail = "with custom prefix and"
[]

[none]
  type = CSVDiff
  input = parent.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=no_prefix_out Transfers/data/prefix='
  csvdiff = no_prefix_out_storage_0002.csv

  detail = "without a prefix."
[]

[complete]
  type = CSVDiff
  input = parent.i
  csvdiff = parent_out_storage.csv

  detail = "can be appended into a single data set or"
[]

[time]
  type = CSVDiff
  input = parent.i
  cli_args = VectorPostprocessors/storage/contains_complete_history=false
  csvdiff = 'parent_out_storage_0000.csv parent_out_storage_0001.csv parent_out_storage_0002.csv'

  detail = "or contain a single file per timestep."
[]