From 2248a602ee806c08a011dc0be7338657da40762e Mon Sep 17 00:00:00 2001 From: Bruno Date: Sun, 17 Jan 2021 12:40:21 -0500 Subject: [PATCH] Step-68 with simplex Add a step-68 test using a simplex mesh Add some small capacities to particle handler to support simplex meshes --- source/particles/particle_handler.cc | 2 +- tests/simplex/step-68.cc | 579 ++++++++++++++++++ ...68.mpirun=1.with_simplex_support=on.output | 51 ++ ...68.mpirun=4.with_simplex_support=on.output | 51 ++ 4 files changed, 682 insertions(+), 1 deletion(-) create mode 100644 tests/simplex/step-68.cc create mode 100644 tests/simplex/step-68.mpirun=1.with_simplex_support=on.output create mode 100644 tests/simplex/step-68.mpirun=4.with_simplex_support=on.output diff --git a/source/particles/particle_handler.cc b/source/particles/particle_handler.cc index 0f5c9b7ed4..7d014f0748 100644 --- a/source/particles/particle_handler.cc +++ b/source/particles/particle_handler.cc @@ -1208,7 +1208,7 @@ namespace Particles if (cell->is_locally_owned()) { std::set cell_to_neighbor_subdomain; - for (const unsigned int v : GeometryInfo::vertex_indices()) + for (const unsigned int v : cell->vertex_indices()) { cell_to_neighbor_subdomain.insert( vertex_to_neighbor_subdomain[cell->vertex_index(v)].begin(), diff --git a/tests/simplex/step-68.cc b/tests/simplex/step-68.cc new file mode 100644 index 0000000000..8bfbc11d63 --- /dev/null +++ b/tests/simplex/step-68.cc @@ -0,0 +1,579 @@ +/* --------------------------------------------------------------------- + * + * Copyright (C) 2020 by the deal.II authors + * + * This file is part of the deal.II library. + * + * The deal.II library is free software; you can use it, redistribute + * it, and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * The full text of the license can be found in the file LICENSE.md at + * the top level directory of deal.II. + * + * --------------------------------------------------------------------- + * + * This test is quasi identical to step-68, with the following exceptions: + * - There is no load balancing + * - A simplex mesh is used for the background mesh + * - The Euler Analytical integration using the analytically defined velocity + * field is not used because it would not test anything relevant + * - Step parameters are hardcoded instead of drawn from a parameter file + * + * Authors: Bruno Blais, Toni El Geitani Nehme, Rene Gassmoeller, Peter Munch + */ + +// @sect3{Include files} + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "../tests.h" + + + +namespace Step68 +{ + using namespace dealii; + + + + template + class Vortex : public Function + { + public: + Vortex() + : Function(dim) + {} + + + virtual void + vector_value(const Point &point, + Vector & values) const override; + }; + + + template + void + Vortex::vector_value(const Point &point, + Vector & values) const + { + const double T = 4; + const double t = this->get_time(); + + const double px = numbers::PI * point(0); + const double py = numbers::PI * point(1); + const double pt = numbers::PI / T * t; + + values[0] = -2 * cos(pt) * pow(sin(px), 2) * sin(py) * cos(py); + values[1] = 2 * cos(pt) * pow(sin(py), 2) * sin(px) * cos(px); + if (dim == 3) + { + values[2] = 0; + } + } + + + + template + class ParticleTracking + { + public: + ParticleTracking(); + void + run(); + + private: + void + generate_particles(); + + void + setup_background_dofs(); + + void + interpolate_function_to_field(); + + void + euler_step_interpolated(const double dt); + void + euler_step_analytical(const double dt); + + void + output_particles(const unsigned int it); + void + output_background(const unsigned int it); + + void + log_particles(); + + MPI_Comm mpi_communicator; + parallel::fullydistributed::Triangulation background_triangulation; + Particles::ParticleHandler particle_handler; + + DoFHandler fluid_dh; + FESystem fluid_fe; + MappingFE mapping; + LinearAlgebra::distributed::Vector velocity_field; + + Vortex velocity; + + // Simulation parameters. In step-68 drawn from prm file, here hardcoded. + std::string output_directory = "./"; + + static constexpr unsigned int velocity_degree = 1; + static constexpr double time_step = 0.004; + static constexpr double final_time = 4.0; + static constexpr unsigned int output_frequency = 1000; + static constexpr unsigned int fluid_refinement = 8; + }; + + template + ParticleTracking::ParticleTracking() + : mpi_communicator(MPI_COMM_WORLD) + , background_triangulation(mpi_communicator) + , fluid_dh(background_triangulation) + , fluid_fe(Simplex::FE_P(velocity_degree), dim) + , mapping(Simplex::FE_P(velocity_degree)) + {} + + // @sect4{Particles generation} + + // This function generates the tracer particles and the background + // triangulation on which these particles evolve. + template + void + ParticleTracking::generate_particles() + { + // We create a hyper cube triangulation which we globally refine. This + // triangulation covers the full trajectory of the particles. + // parallel::distributed::Triangulation tria_pdt(mpi_communicator); + // + // GridGenerator::hyper_cube(tria_pdt, 0, 1); + // tria_pdt.refine_global(fluid_refinement); + + Triangulation temporary_triangulation; + GridGenerator::subdivided_hyper_cube_with_simplices(temporary_triangulation, + fluid_refinement); + + + // extract relevant information from distributed triangulation + auto construction_data = TriangulationDescription::Utilities:: + create_description_from_triangulation(temporary_triangulation, + mpi_communicator); + background_triangulation.create_triangulation(construction_data); + + + // This initializes the background triangulation where the particles are + // living and the number of properties of the particles. + particle_handler.initialize(background_triangulation, mapping, 1 + dim); + + // We create a particle triangulation which is solely used to generate + // the points which will be used to insert the particles. This + // triangulation is a hyper shell which is offset from the + // center of the simulation domain. This will be used to generate a + // disk filled with particles which will allow an easy monitoring + // of the motion due to the vortex. + Point center; + center[0] = 0.5; + center[1] = 0.75; + if (dim == 3) + center[2] = 0.5; + + const double outer_radius = 0.15; + const double inner_radius = 0.01; + + Triangulation temporary_quad_particle_triangulation; + + GridGenerator::hyper_shell(temporary_quad_particle_triangulation, + center, + inner_radius, + outer_radius, + 6); + + + Triangulation temporary_tri_particle_triangulation; + GridGenerator::convert_hypercube_to_simplex_mesh( + temporary_quad_particle_triangulation, + temporary_tri_particle_triangulation); + + + // extract relevant information from distributed triangulation + auto particle_construction_data = TriangulationDescription::Utilities:: + create_description_from_triangulation( + temporary_tri_particle_triangulation, mpi_communicator); + + + + parallel::fullydistributed::Triangulation particle_triangulation( + mpi_communicator); + particle_triangulation.create_triangulation(particle_construction_data); + + // We generate the necessary bounding boxes for the particles generator. + // These bounding boxes are required to quickly identify in which + // process's subdomain the inserted particle lies, and which cell owns it. + + std::vector> all_boxes; + all_boxes.reserve(background_triangulation.n_locally_owned_active_cells()); + for (const auto cell : background_triangulation.active_cell_iterators()) + if (cell->is_locally_owned()) + all_boxes.emplace_back(cell->bounding_box()); + const auto tree = pack_rtree(all_boxes); + const auto local_boxes = extract_rtree_level(tree, 1); + + std::vector>> global_bounding_boxes; + global_bounding_boxes = + Utilities::MPI::all_gather(mpi_communicator, local_boxes); + + // We generate an empty vector of properties. We will attribute the + // properties to the particles once they are generated. + std::vector> properties( + particle_triangulation.n_locally_owned_active_cells(), + std::vector(dim + 1, 0.)); + + // We generate the particles at the position of a single + // point quadrature. Consequently, one particle will be generated + // at the centroid of each cell. + Simplex::QGauss quadrature_formula(1); + + Particles::Generators::quadrature_points(particle_triangulation, + quadrature_formula, + global_bounding_boxes, + particle_handler, + mapping, + properties); + + if (Utilities::MPI::this_mpi_process(mpi_communicator) == 0) + deallog << "Number of particles inserted: " + << particle_handler.n_global_particles() << std::endl; + } + + + + // @sect4{Background DOFs and interpolation} + + // This function sets up the background degrees of freedom used for the + // velocity interpolation and allocates the field vector where the entire + // solution of the velocity field is stored. + template + void + ParticleTracking::setup_background_dofs() + { + fluid_dh.distribute_dofs(fluid_fe); + const IndexSet locally_owned_dofs = fluid_dh.locally_owned_dofs(); + IndexSet locally_relevant_dofs; + DoFTools::extract_locally_relevant_dofs(fluid_dh, locally_relevant_dofs); + + velocity_field.reinit(locally_owned_dofs, + locally_relevant_dofs, + mpi_communicator); + } + + + + // This function takes care of interpolating the + // vortex velocity field to the field vector. This is achieved rather easily + // by using the VectorTools::interpolate() function. + template + void + ParticleTracking::interpolate_function_to_field() + { + velocity_field.zero_out_ghosts(); + VectorTools::interpolate(mapping, fluid_dh, velocity, velocity_field); + velocity_field.update_ghost_values(); + } + + + + // @sect4{Time integration of the trajectories} + + // In contrast to the previous function in this function we + // integrate the particle trajectories by interpolating the value of + // the velocity field at the degrees of freedom to the position of + // the particles. + template + void + ParticleTracking::euler_step_interpolated(const double dt) + { + Vector local_dof_values(fluid_fe.dofs_per_cell); + + // We loop over all the local particles. Although this could be achieved + // directly by looping over all the cells, this would force us + // to loop over numerous cells which do not contain particles. + // Rather, we loop over all the particles, but, we get the reference + // of the cell in which the particle lies and then loop over all particles + // within that cell. This enables us to gather the values of the velocity + // out of the `velocity_field` vector once and use them for all particles + // that lie within the cell. + auto particle = particle_handler.begin(); + while (particle != particle_handler.end()) + { + const auto cell = + particle->get_surrounding_cell(background_triangulation); + const auto dh_cell = + typename DoFHandler::cell_iterator(*cell, &fluid_dh); + + dh_cell->get_dof_values(velocity_field, local_dof_values); + + // Next, compute the velocity at the particle locations by evaluating + // the finite element solution at the position of the particles. + // This is essentially an optimized version of the particle advection + // functionality in step 19, but instead of creating quadrature + // objects and FEValues objects for each cell, we do the + // evaluation by hand, which is somewhat more efficient and only + // matters for this tutorial, because the particle work is the + // dominant cost of the whole program. + const auto pic = particle_handler.particles_in_cell(cell); + Assert(pic.begin() == particle, ExcInternalError()); + for (auto &p : pic) + { + const Point reference_location = p.get_reference_location(); + Tensor<1, dim> particle_velocity; + for (unsigned int j = 0; j < fluid_fe.dofs_per_cell; ++j) + { + const auto comp_j = fluid_fe.system_to_component_index(j); + + particle_velocity[comp_j.first] += + fluid_fe.shape_value(j, reference_location) * + local_dof_values[j]; + } + + Point particle_location = particle->get_location(); + for (int d = 0; d < dim; ++d) + particle_location[d] += particle_velocity[d] * dt; + p.set_location(particle_location); + + // Again, we store the particle velocity and the processor id in the + // particle properties for visualization purposes. + ArrayView properties = p.get_properties(); + for (int d = 0; d < dim; ++d) + properties[d] = particle_velocity[d]; + + properties[dim] = + Utilities::MPI::this_mpi_process(mpi_communicator); + + ++particle; + } + } + } + + + + // @sect4{Data output} + + // The next two functions take care of writing both the particles + // and the background mesh to vtu with a pvtu record. This ensures + // that the simulation results can be visualized when the simulation is + // launched in parallel. + template + void + ParticleTracking::output_particles(const unsigned int it) + { + Particles::DataOut particle_output; + + std::vector solution_names(dim, "velocity"); + solution_names.push_back("process_id"); + + std::vector + data_component_interpretation( + dim, DataComponentInterpretation::component_is_part_of_vector); + data_component_interpretation.push_back( + DataComponentInterpretation::component_is_scalar); + + particle_output.build_patches(particle_handler, + solution_names, + data_component_interpretation); + const std::string output_folder(output_directory); + const std::string file_name("interpolated-particles"); + + particle_output.write_vtu_with_pvtu_record( + output_folder, file_name, it, mpi_communicator, 6); + } + + template + void + ParticleTracking::log_particles() + { + if (Utilities::MPI::this_mpi_process(mpi_communicator) == 0) + deallog << "Particles location" << std::endl; + for (unsigned int proc = 0; + proc < Utilities::MPI::n_mpi_processes(mpi_communicator); + ++proc) + { + if (Utilities::MPI::this_mpi_process(mpi_communicator) == proc) + { + for (auto part : particle_handler) + { + deallog << part.get_location() << std::endl; + } + } + MPI_Barrier(mpi_communicator); + } + } + + + + template + void + ParticleTracking::output_background(const unsigned int it) + { + std::vector solution_names(dim, "velocity"); + std::vector + data_component_interpretation( + dim, DataComponentInterpretation::component_is_part_of_vector); + + DataOut data_out; + + // Attach the solution data to data_out object + data_out.attach_dof_handler(fluid_dh); + data_out.add_data_vector(velocity_field, + solution_names, + DataOut::type_dof_data, + data_component_interpretation); + Vector subdomain(background_triangulation.n_active_cells()); + for (unsigned int i = 0; i < subdomain.size(); ++i) + subdomain(i) = background_triangulation.locally_owned_subdomain(); + data_out.add_data_vector(subdomain, "subdomain"); + + data_out.build_patches(mapping); + + const std::string output_folder(output_directory); + const std::string file_name("background"); + + data_out.write_vtu_with_pvtu_record( + output_folder, file_name, it, mpi_communicator, 6); + } + + + template + void + ParticleTracking::run() + { + DiscreteTime discrete_time(0, final_time, time_step); + + generate_particles(); + + // We set the initial property of the particles by doing an + // explicit Euler iteration with a time-step of 0 both in the case + // of the analytical and the interpolated approach. + setup_background_dofs(); + interpolate_function_to_field(); + euler_step_interpolated(0.); + + output_particles(discrete_time.get_step_number()); + output_background(discrete_time.get_step_number()); + + // The particles are advected by looping over time. + while (!discrete_time.is_at_end()) + { + discrete_time.advance_time(); + velocity.set_time(discrete_time.get_previous_time()); + + interpolate_function_to_field(); + euler_step_interpolated(discrete_time.get_previous_step_size()); + + // After the particles have been moved, it is necessary to identify + // in which cell they now reside. This is achieved by calling + // sort_particles_into_subdomains_and_cells + particle_handler.sort_particles_into_subdomains_and_cells(); + + if ((discrete_time.get_step_number() % output_frequency) == 0) + { + output_particles(discrete_time.get_step_number()); + output_background(discrete_time.get_step_number()); + } + } + log_particles(); + } + +} // namespace Step68 + + + +// @sect3{The main() function} + +// The remainder of the code, the `main()` function, is standard. +// We note that we run the particle tracking with the analytical velocity +// and the interpolated velocity and produce both results +int +main(int argc, char *argv[]) +{ + using namespace Step68; + using namespace dealii; + Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv, 1); + + initlog(); + deallog.depth_console(1); + + + try + { + Step68::ParticleTracking<2> particle_tracking; + particle_tracking.run(); + } + catch (std::exception &exc) + { + std::cerr << std::endl + << std::endl + << "----------------------------------------------------" + << std::endl; + std::cerr << "Exception on processing: " << std::endl + << exc.what() << std::endl + << "Aborting!" << std::endl + << "----------------------------------------------------" + << std::endl; + + return 1; + } + catch (...) + { + std::cerr << std::endl + << std::endl + << "----------------------------------------------------" + << std::endl; + std::cerr << "Unknown exception!" << std::endl + << "Aborting!" << std::endl + << "----------------------------------------------------" + << std::endl; + return 1; + } + + return 0; +} diff --git a/tests/simplex/step-68.mpirun=1.with_simplex_support=on.output b/tests/simplex/step-68.mpirun=1.with_simplex_support=on.output new file mode 100644 index 0000000000..427bb9e901 --- /dev/null +++ b/tests/simplex/step-68.mpirun=1.with_simplex_support=on.output @@ -0,0 +1,51 @@ + +DEAL::Number of particles inserted: 48 +DEAL::Particles location +DEAL::0.366577 0.681804 +DEAL::0.398361 0.691931 +DEAL::0.408783 0.708456 +DEAL::0.423868 0.706937 +DEAL::0.433550 0.698432 +DEAL::0.444141 0.713765 +DEAL::0.454362 0.697548 +DEAL::0.453680 0.706030 +DEAL::0.464383 0.709034 +DEAL::0.469644 0.709187 +DEAL::0.484716 0.715907 +DEAL::0.499111 0.711062 +DEAL::0.498868 0.710828 +DEAL::0.517719 0.673900 +DEAL::0.503282 0.705982 +DEAL::0.520884 0.707622 +DEAL::0.524944 0.701767 +DEAL::0.535494 0.683627 +DEAL::0.532509 0.716124 +DEAL::0.549286 0.705238 +DEAL::0.563363 0.673326 +DEAL::0.573466 0.689253 +DEAL::0.618983 0.657786 +DEAL::0.628548 0.625331 +DEAL::0.691290 0.744340 +DEAL::0.379293 0.831136 +DEAL::0.391377 0.827051 +DEAL::0.409176 0.839420 +DEAL::0.448930 0.832650 +DEAL::0.465573 0.816761 +DEAL::0.476410 0.833958 +DEAL::0.494476 0.826914 +DEAL::0.493642 0.819960 +DEAL::0.518139 0.836076 +DEAL::0.559598 0.806751 +DEAL::0.568815 0.811801 +DEAL::0.571248 0.776918 +DEAL::0.586805 0.806620 +DEAL::0.591790 0.794930 +DEAL::0.591050 0.794375 +DEAL::0.607814 0.783764 +DEAL::0.613584 0.807264 +DEAL::0.629515 0.787985 +DEAL::0.647557 0.779179 +DEAL::0.660445 0.781133 +DEAL::0.655199 0.775754 +DEAL::0.698558 0.758550 +DEAL::0.631046 0.755799 diff --git a/tests/simplex/step-68.mpirun=4.with_simplex_support=on.output b/tests/simplex/step-68.mpirun=4.with_simplex_support=on.output new file mode 100644 index 0000000000..427bb9e901 --- /dev/null +++ b/tests/simplex/step-68.mpirun=4.with_simplex_support=on.output @@ -0,0 +1,51 @@ + +DEAL::Number of particles inserted: 48 +DEAL::Particles location +DEAL::0.366577 0.681804 +DEAL::0.398361 0.691931 +DEAL::0.408783 0.708456 +DEAL::0.423868 0.706937 +DEAL::0.433550 0.698432 +DEAL::0.444141 0.713765 +DEAL::0.454362 0.697548 +DEAL::0.453680 0.706030 +DEAL::0.464383 0.709034 +DEAL::0.469644 0.709187 +DEAL::0.484716 0.715907 +DEAL::0.499111 0.711062 +DEAL::0.498868 0.710828 +DEAL::0.517719 0.673900 +DEAL::0.503282 0.705982 +DEAL::0.520884 0.707622 +DEAL::0.524944 0.701767 +DEAL::0.535494 0.683627 +DEAL::0.532509 0.716124 +DEAL::0.549286 0.705238 +DEAL::0.563363 0.673326 +DEAL::0.573466 0.689253 +DEAL::0.618983 0.657786 +DEAL::0.628548 0.625331 +DEAL::0.691290 0.744340 +DEAL::0.379293 0.831136 +DEAL::0.391377 0.827051 +DEAL::0.409176 0.839420 +DEAL::0.448930 0.832650 +DEAL::0.465573 0.816761 +DEAL::0.476410 0.833958 +DEAL::0.494476 0.826914 +DEAL::0.493642 0.819960 +DEAL::0.518139 0.836076 +DEAL::0.559598 0.806751 +DEAL::0.568815 0.811801 +DEAL::0.571248 0.776918 +DEAL::0.586805 0.806620 +DEAL::0.591790 0.794930 +DEAL::0.591050 0.794375 +DEAL::0.607814 0.783764 +DEAL::0.613584 0.807264 +DEAL::0.629515 0.787985 +DEAL::0.647557 0.779179 +DEAL::0.660445 0.781133 +DEAL::0.655199 0.775754 +DEAL::0.698558 0.758550 +DEAL::0.631046 0.755799 -- 2.39.5