From: Matthias Maier Date: Sat, 23 May 2020 07:14:00 +0000 (-0500) Subject: CMake: Add a bundled taskflow library and facility to search for external one X-Git-Tag: v9.3.0-rc1~1546^2~5 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1eb405d189c890c2dadd370062556e0353e2c63e;p=dealii.git CMake: Add a bundled taskflow library and facility to search for external one --- diff --git a/bundled/CMakeLists.txt b/bundled/CMakeLists.txt index a99cc2e90b..ae7ea16927 100644 --- a/bundled/CMakeLists.txt +++ b/bundled/CMakeLists.txt @@ -37,7 +37,15 @@ IF(FEATURE_BOOST_BUNDLED_CONFIGURED) ENDIF() -IF(FEATURE_THREADS_BUNDLED_CONFIGURED) +IF(FEATURE_TASKFLOW_BUNDLED_CONFIGURED) + INSTALL(DIRECTORY ${TASKFLOW_FOLDER}/include/taskflow + DESTINATION ${DEAL_II_INCLUDE_RELDIR}/deal.II/bundled + COMPONENT library + ) +ENDIF() + + +IF(FEATURE_TBB_BUNDLED_CONFIGURED) INSTALL(DIRECTORY ${TBB_FOLDER}/include/tbb DESTINATION ${DEAL_II_INCLUDE_RELDIR}/deal.II/bundled COMPONENT library diff --git a/bundled/README.md b/bundled/README.md index c8de40dca6..4cac7caa9a 100644 --- a/bundled/README.md +++ b/bundled/README.md @@ -19,6 +19,15 @@ and licensed under the Boost Software License Version 1.0. See A full version of the library can be downloaded at http://www.boost.org/. +cpp-taskflow-* +-------------- + +Contains the cpp-taskflow project licensed under the MIT license. + +A full version of the FunctionParser project can be downloaded at +https://cpp-taskflow.github.io/#/ + + muparser_* ---------- diff --git a/bundled/cpp-taskflow/LICENSE b/bundled/cpp-taskflow/LICENSE new file mode 100644 index 0000000000..889eb137e4 --- /dev/null +++ b/bundled/cpp-taskflow/LICENSE @@ -0,0 +1,24 @@ +MIT License + +Copyright (c) 2018-2020 T.-W. Huang, C.-X. Lin, G. Guo, and M. Wong + +University of Utah, Salt Lake City, UT, USA +University of Illinois at Urbana-Champaign, IL, USA + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bundled/cpp-taskflow/README.md b/bundled/cpp-taskflow/README.md new file mode 100644 index 0000000000..07fa377a85 --- /dev/null +++ b/bundled/cpp-taskflow/README.md @@ -0,0 +1,821 @@ +# Cpp-Taskflow + +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/bb04cb8e4aca401b8206c054e79fd5e3)](https://app.codacy.com/app/tsung-wei-huang/cpp-taskflow?utm_source=github.com&utm_medium=referral&utm_content=cpp-taskflow/cpp-taskflow&utm_campaign=Badge_Grade_Dashboard) +[![Linux Build Status](https://travis-ci.com/cpp-taskflow/cpp-taskflow.svg?branch=master)](https://travis-ci.com/cpp-taskflow/cpp-taskflow) +[![Windows Build status](https://ci.appveyor.com/api/projects/status/te9bjp4yfhq7f8hq?svg=true)](https://ci.appveyor.com/project/TsungWeiHuang/cpp-taskflow) +[![Wiki](image/api-doc.svg)][wiki] +[![TFProf](image/tfprof.svg)](https://cpp-taskflow.github.io/tfprof/) +[![Cite](image/cite-arXiv.svg)](https://arxiv.org/abs/2004.10908v2) + +Cpp-Taskflow helps you quickly write parallel programs with high performance scalability +and simultaneous high productivity. + +# Why Cpp-Taskflow? + +Cpp-Taskflow is faster, more expressive, and easier for drop-in integration +than many of existing task programming frameworks +in handling complex parallel workloads. + +![](image/performance.png) + +Cpp-Taskflow lets you quickly implement task decomposition strategies +that incorporate both regular and irregular compute patterns, +together with an efficient *work-stealing* scheduler to optimize your multithreaded performance. + +| [Static Tasking](#get-started-with-cpp-taskflow) | [Dynamic Tasking](#dynamic-tasking) | +| :------------: | :-------------: | +| ![](image/static_graph.svg) | | + +Cpp-Taskflow supports conditional tasking for you to make rapid control-flow decisions +across dependent tasks to implement cycles and conditions that were otherwise difficult to do +with existing tools. + +| [Conditional Tasking](#conditional-tasking) | +| :-----------------: | +| ![](image/condition.svg) | + +Cpp-Taskflow is composable. You can create large parallel graphs through +composition of modular and reusable blocks that are easier to optimize +at an individual scope. + +| [Taskflow Composition](#composable-tasking) | +| :---------------: | +|![](image/framework.svg)| + +Cpp-Taskflow supports heterogeneous tasking for you to +accelerate a wide range of scientific computing applications +by harnessing the power of CPU-GPU collaborative computing. + +| [Concurrent CPU-GPU Tasking](#concurrent-cpu-gpu-tasking) | +| :-----------------: | +| ![](image/cudaflow.svg) | + +We are committed to support trustworthy developments for both academic and industrial research projects +in parallel computing. Check out [Who is Using Cpp-Taskflow](#who-is-using-cpp-taskflow) and what our users say: + ++ *"Cpp-Taskflow is the cleanest Task API I've ever seen." [Damien Hocking @Corelium Inc](http://coreliuminc.com)* ++ *"Cpp-Taskflow has a very simple and elegant tasking interface. The performance also scales very well." [Glen Fraser][totalgee]* ++ *"Cpp-Taskflow lets me handle parallel processing in a smart way." [Hayabusa @Cpp-Learning](https://cpp-learning.com/cpp-taskflow/)* ++ *"Cpp-Taskflow improves the throughput of our graph engine in just a few hours of coding." [Jean-Michaël @KDAB](https://ossia.io/)* ++ *"Best poster award for open-source parallel programming library." [Cpp Conference 2018][Cpp Conference 2018]* ++ *"Second Prize of Open-source Software Competition." [ACM Multimedia Conference 2019](https://tsung-wei-huang.github.io/img/mm19-ossc-award.jpg)* + +See a quick [presentation][Presentation] and +visit the [documentation][wiki] to learn more about Cpp-Taskflow. +Technical details can be referred to our [arXiv paper](https://arxiv.org/abs/2004.10908v2). + +# Table of Contents + +* [Get Started with Cpp-Taskflow](#get-started-with-cpp-taskflow) +* [Create a Taskflow Application](#create-a-taskflow-application) + * [Step 1: Create a Taskflow](#step-1-create-a-taskflow) + * [Step 2: Define Task Dependencies](#step-2-define-task-dependencies) + * [Step 3: Execute a Taskflow](#step-3-execute-a-taskflow) +* [Dynamic Tasking](#dynamic-tasking) +* [Conditional Tasking](#conditional-tasking) + * [Step 1: Create a Condition Task](#step-1-create-a-condition-task) + * [Step 2: Scheduling Rules for Condition Tasks](#step-2-scheduling-rules-for-condition-tasks) +* [Composable Tasking](#composable-tasking) +* [Concurrent CPU-GPU Tasking](#concurrent-cpu-gpu-tasking) + * [Step 1: Create a cudaFlow](#step-1-create-a-cudaflow) + * [Step 2: Compile and Execute a cudaFlow](#step-2-compile-and-execute-a-cudaflow) +* [Visualize a Taskflow Graph](#visualize-a-taskflow-graph) +* [Monitor Thread Activities](#monitor-thread-activities) +* [API Reference](#api-reference) +* [System Requirements](#system-requirements) +* [Compile Unit Tests, Examples, and Benchmarks](#compile-unit-tests-examples-and-benchmarks) +* [Who is Using Cpp-Taskflow?](#who-is-using-cpp-taskflow) + + +# Get Started with Cpp-Taskflow + +The following example [simple.cpp](./examples/simple.cpp) shows the basic Cpp-Taskflow API +you need in most applications. + +```cpp +#include // Cpp-Taskflow is header-only + +int main(){ + + tf::Executor executor; + tf::Taskflow taskflow; + + auto [A, B, C, D] = taskflow.emplace( + [] () { std::cout << "TaskA\n"; }, // task dependency graph + [] () { std::cout << "TaskB\n"; }, // + [] () { std::cout << "TaskC\n"; }, // +---+ + [] () { std::cout << "TaskD\n"; } // +---->| B |-----+ + ); // | +---+ | + // +---+ +-v-+ + A.precede(B); // A runs before B // | A | | D | + A.precede(C); // A runs before C // +---+ +-^-+ + B.precede(D); // B runs before D // | +---+ | + C.precede(D); // C runs before D // +---->| C |-----+ + // +---+ + executor.run(taskflow).wait(); + + return 0; +} +``` + +Compile and run the code with the following commands: + +```bash +~$ g++ simple.cpp -I path/to/include/taskflow/ -std=c++17 -O2 -lpthread -o simple +~$ ./simple +TaskA +TaskC <-- concurrent with TaskB +TaskB <-- concurrent with TaskC +TaskD +``` + +# Create a Taskflow Application + +Cpp-Taskflow defines a very expressive API to create task dependency graphs. +Most applications are developed through the following three steps: + +## Step 1: Create a Taskflow + +Create a taskflow object to build a task dependency graph: + +```cpp +tf::Taskflow taskflow; +``` + +A task is a callable object for which [std::invoke][std::invoke] is applicable. +Use the method `emplace` to create a task: + +```cpp +tf::Task A = taskflow.emplace([](){ std::cout << "Task A\n"; }); +``` + +## Step 2: Define Task Dependencies + +You can add dependency links between tasks to enforce one task to run before or +after another. + +```cpp +A.precede(B); // A runs before B. +``` + +## Step 3: Execute a Taskflow + +To execute a taskflow, you need to create an *executor*. +An executor manages a set of worker threads to execute a taskflow +through an efficient *work-stealing* algorithm. + +```cpp +tf::Executor executor; +``` + +The executor provides a rich set of methods to run a taskflow. +You can run a taskflow multiple times, or until a stopping criteria is met. +These methods are non-blocking with a [std::future][std::future] return +to let you query the execution status. +Executor is *thread-safe*. + +```cpp +executor.run(taskflow); // runs the taskflow once +executor.run_n(taskflow, 4); // runs the taskflow four times + +// keeps running the taskflow until the predicate becomes true +executor.run_until(taskflow, [counter=4](){ return --counter == 0; } ); +``` + + +You can call `wait_for_all` to block the executor until all associated taskflows complete. + +```cpp +executor.wait_for_all(); // block until all associated tasks finish +``` + +Notice that the executor does not own any taskflow. +It is your responsibility to keep a taskflow alive during its execution, +or it can result in undefined behavior. +In most applications, you need only one executor to run multiple taskflows +each representing a specific part of your parallel decomposition. + +
[↑]
+ +# Dynamic Tasking + +Another powerful feature of Taskflow is *dynamic* tasking. +Dynamic tasks are those tasks created during the execution of a taskflow. +These tasks are spawned by a parent task and are grouped together to a *subflow* graph. +To create a subflow for dynamic tasking, +emplace a callable with one argument of type `tf::Subflow`. + + + +```cpp +// create three regular tasks +tf::Task A = tf.emplace([](){}).name("A"); +tf::Task C = tf.emplace([](){}).name("C"); +tf::Task D = tf.emplace([](){}).name("D"); + +// create a subflow graph (dynamic tasking) +tf::Task B = tf.emplace([] (tf::Subflow& subflow) { + tf::Task B1 = subflow.emplace([](){}).name("B1"); + tf::Task B2 = subflow.emplace([](){}).name("B2"); + tf::Task B3 = subflow.emplace([](){}).name("B3"); + B1.precede(B3); + B2.precede(B3); +}).name("B"); + +A.precede(B); // B runs after A +A.precede(C); // C runs after A +B.precede(D); // D runs after B +C.precede(D); // D runs after C +``` + +By default, a subflow graph joins its parent node. +This ensures a subflow graph finishes before the successors of +its parent task. +You can disable this feature by calling `subflow.detach()`. +For example, detaching the above subflow will result in the following execution flow: + + + +```cpp +// create a "detached" subflow graph (dynamic tasking) +tf::Task B = tf.emplace([] (tf::Subflow& subflow) { + tf::Task B1 = subflow.emplace([](){}).name("B1"); + tf::Task B2 = subflow.emplace([](){}).name("B2"); + tf::Task B3 = subflow.emplace([](){}).name("B3"); + + B1.precede(B3); + B2.precede(B3); + + // detach the subflow to form a parallel execution line + subflow.detach(); +}).name("B"); +``` + +A subflow can be nested or recursive. You can create another subflow from +the execution of a subflow and so on. + +
[↑]
+ +# Conditional Tasking + +Taskflow supports *conditional tasking* for users to implement *dynamic* and *cyclic* control flows. +You can create highly versatile and efficient parallel patterns through condition tasks. + +## Step 1: Create a Condition Task + +A *condition task* evalutes a set of instructions and returns an integer index +of the next immediate successor to execute. +The index is defined with respect to the order of its successor construction. + + + +```cpp +tf::Task init = tf.emplace([](){ }).name("init"); +tf::Task stop = tf.emplace([](){ }).name("stop"); + +// creates a condition task that returns 0 or 1 +tf::Task cond = tf.emplace([](){ + std::cout << "flipping a coin\n"; + return rand() % 2; +}).name("cond"); + +// creates a feedback loop +init.precede(cond); +cond.precede(cond, stop); // cond--0-->cond, cond--1-->stop + +executor.run(tf).wait(); +``` + +If the return value from `cond` is 0, it loops back to itself, or otherwise to `stop`. +Cpp-Taskflow terms the preceding link from a condition task a *weak dependency* +(dashed lines above). +Others are *strong depedency* (solid lines above). + + +## Step 2: Scheduling Rules for Condition Tasks + +When you submit a taskflow to an executor, +the scheduler starts with tasks of *zero dependency* (both weak and strong dependencies) +and continues to execute successive tasks whenever *strong dependencies* are met. +However, +the scheduler skips this rule for a condition task and jumps directly to its successor +indexed by the return value. + +![](image/conditional-tasking-rules.svg) + +It is users' responsibility to ensure a taskflow is properly conditioned. +Top things to avoid include no source tasks to start with and task race. +The figure shows common pitfalls and their remedies. +In the risky scenario, task X may not be raced if P and M is exclusively +branching to X. + +![](image/conditional-tasking-pitfalls.svg) + + +A good practice for avoiding mistakes of conditional tasking is to infer the execution flow of your graphs based on our scheduling rules. +Make sure there is no task race. + +
[↑]
+ + + +# Composable Tasking + +A powerful feature of `tf::Taskflow` is composability. +You can create multiple task graphs from different parts of your workload +and use them to compose a large graph through the `composed_of` method. + + + + +```cpp +tf::Taskflow f1, f2; + +auto [f1A, f1B] = f1.emplace( + []() { std::cout << "Task f1A\n"; }, + []() { std::cout << "Task f1B\n"; } +); +auto [f2A, f2B, f2C] = f2.emplace( + []() { std::cout << "Task f2A\n"; }, + []() { std::cout << "Task f2B\n"; }, + []() { std::cout << "Task f2C\n"; } +); +auto f1_module_task = f2.composed_of(f1); + +f1_module_task.succeed(f2A, f2B) + .precede(f2C); +``` + +Similarly, `composed_of` returns a task handle and you can use +`precede` to create dependencies. +You can compose a taskflow from multiple taskflows and use the result +to compose a larger taskflow and so on. + +
[↑]
+ +# Concurrent CPU-GPU Tasking + +Cpp-Taskflow enables concurrent CPU-GPU tasking by leveraging +[Nvidia CUDA Toolkit][cuda-toolkit]. +You can harness the power of CPU-GPU collaborative computing +to implement heterogeneous decomposition algorithms. + +## Step 1: Create a cudaFlow + +A `tf::cudaFlow` is a graph object created at runtime +similar to dynamic tasking. +It manages a task node in a taskflow and associates it +with a [CUDA Graph][cudaGraph]. +To create a cudaFlow, emplace a callable with an argument +of type `tf::cudaFlow`. + + + +```cpp +tf::Taskflow taskflow; +tf::Executor executor; + +const unsigned N = 1<<20; // size of the vector +std::vector hx(N, 1.0f), hy(N, 2.0f); // x and y vectors at host +float *dx{nullptr}, *dy{nullptr}; // x and y vectors at device + +tf::Task allocate_x = taskflow.emplace([&](){ cudaMalloc(&dx, N*sizeof(float));}); +tf::Task allocate_y = taskflow.emplace([&](){ cudaMalloc(&dy, N*sizeof(float));}); +tf::Task cudaflow = taskflow.emplace([&](tf::cudaFlow& cf) { + tf::cudaTask h2d_x = cf.copy(dx, hx.data(), N); // host-to-device x data transfer + tf::cudaTask h2d_y = cf.copy(dy, hy.data(), N); // host-to-device y data transfer + tf::cudaTask d2h_x = cf.copy(hx.data(), dx, N); // device-to-host x data transfer + tf::cudaTask d2h_y = cf.copy(hy.data(), dy, N); // device-to-host y data transfer + // launch saxpy<<<(N+255)/256, 256, 0>>>(N, 2.0f, dx, dy) + tf::cudaTask kernel = cf.kernel((N+255)/256, 256, 0, saxpy, N, 2.0f, dx, dy); + kernel.succeed(h2d_x, h2d_y) + .precede(d2h_x, d2h_y); +}); +cudaflow.succeed(allocate_x, allocate_y); // overlap data allocations + +executor.run(taskflow).wait(); +``` + +Assume our kernel implements the canonical saxpy operation +(single-precision A·X Plus Y) using the CUDA syntax. + + + +```cpp +// saxpy (single-precision A·X Plus Y) kernel +__global__ void saxpy( + int n, float a, float *x, float *y +) { + // get the thread index + int i = blockIdx.x*blockDim.x + threadIdx.x; + + if (i < n) { + y[i] = a*x[i] + y[i]; + } +} +``` + + + +## Step 2: Compile and Execute a cudaFlow + +Name you source with the extension `.cu`, let's say `saxpy.cu`, +and compile it through [nvcc][nvcc]: + +```bash +~$ nvcc saxpy.cu -I path/to/include/taskflow -O2 -o saxpy +~$ ./saxpy +``` + +Our source autonomously enables cudaFlow for compilers that support +CUDA. + +
[↑]
+ +# Visualize a Taskflow Graph + +You can dump a taskflow through a `std::ostream` +in [GraphViz][GraphViz] format using the method `dump`. +There are a number of free [GraphViz tools][AwesomeGraphViz] you could find online to visualize your Taskflow graph. + + + +```cpp +tf::Taskflow taskflow; +tf::Task A = taskflow.emplace([] () {}).name("A"); +tf::Task B = taskflow.emplace([] () {}).name("B"); +tf::Task C = taskflow.emplace([] () {}).name("C"); +tf::Task D = taskflow.emplace([] () {}).name("D"); +tf::Task E = taskflow.emplace([] () {}).name("E"); +A.precede(B, C, E); +C.precede(D); +B.precede(D, E); + +taskflow.dump(std::cout); // dump the graph in DOT to std::cout +``` + +When you have tasks that are created at runtime (e.g., subflow, cudaFlow), +you need to execute the graph first to spawn these tasks +and dump the entire graph. + + + +```cpp +tf::Executor executor; +tf::Taskflow taskflow; + +tf::Task A = taskflow.emplace([](){}).name("A"); + +// create a subflow of two tasks B1->B2 +tf::Task B = taskflow.emplace([] (tf::Subflow& subflow) { + tf::Task B1 = subflow.emplace([](){}).name("B1"); + tf::Task B2 = subflow.emplace([](){}).name("B2"); + B1.precede(B2); +}).name("B"); + +A.precede(B); + +executor.run(tf).wait(); // run the taskflow to spawn subflows +tf.dump(std::cout); // dump the graph including dynamic tasks +``` + +
[↑]
+ + + + +# Monitor Thread Activities + +Cpp-Taskflow Profiler ([TFProf](https://github.com/cpp-taskflow/tfprof)) +provides the visualization and tooling needed for profiling cpp-taskflow programs. + +

+ + + +

+ +
[↑]
+ +# API Reference + +The official [documentation][wiki] explains a complete list of +Cpp-Taskflow API. +Here, we highlight commonly used methods. + +## Taskflow API + +The class `tf::Taskflow` is the main place to create a task dependency graph. +The table below summarizes a list of commonly used methods. + +| Method | Argument | Return | Description | +| -------- | --------- | ------- | ----------- | +| emplace | callables | tasks | creates a task with a given callable(s) | +| placeholder | none | task | inserts a node without any work; work can be assigned later | +| parallel_for | beg, end, callable, chunk | task pair | concurrently applies the callable chunk by chunk to the result of dereferencing every iterator in the range | +| parallel_for | beg, end, step, callable, chunk | task pair | concurrently applies the callable chunk by chunk to an index-based range with a step size | +| num_workers | none | size | queries the number of working threads in the pool | +| dump | ostream | none | dumps the taskflow to an output stream in GraphViz format | + +### *emplace/placeholder* + +You can use `emplace` to create a task from a target callable. + +```cpp +tf::Task task = tf.emplace([] () { std::cout << "my task\n"; }); +``` + +When a task cannot be determined beforehand, you can create a placeholder and assign the callable later. + +```cpp +tf::Task A = tf.emplace([](){}); +tf::Task B = tf.placeholder(); +A.precede(B); +B.work([](){ /* do something */ }); +``` + +### *parallel_for* + +The method `parallel_for` creates a subgraph that applies the callable to each item in the given range of a container. + + + +```cpp +auto v = {'A', 'B', 'C', 'D'}; +auto [S, T] = tf.parallel_for( + v.begin(), // iterator to the beginning + v.end(), // iterator to the end + [] (int i) { + std::cout << "parallel " << i << '\n'; + } +); +// add dependencies via S and T. +``` + +You can specify a *chunk* size (default one) in the last argument to force a task to include a certain number of items. + + + +```cpp +auto v = {'A', 'B', 'C', 'D'}; +auto [S, T] = tf.parallel_for( + v.begin(), // iterator to the beginning + v.end(), // iterator to the end + [] (int i) { + std::cout << "AB and CD run in parallel" << '\n'; + }, + 2 // at least two items at a time +); +``` + +In addition to iterator-based construction, +`parallel_for` has another overload of index-based loop. +The first three argument of this overload indicates +starting index, ending index (exclusive), and step size. + +```cpp +// [0, 11) with a step size of 2 +auto [S, T] = tf.parallel_for( + 0, 11, 2, + [] (int i) { + std::cout << "parallel_for on index " << i << std::endl; + }, + 2 // at least two items at a time +); +// will print 0, 2, 4, 6, 8, 10 (three partitions, {0, 2}, {4, 6}, {8, 10}) +``` + +## Task API + +Each time you create a task, the taskflow object adds a node to the present task dependency graph +and return a *task handle* to you. +A task handle is a lightweight object that defines a set of methods for users to +access and modify the attributes of the associated task. +The table below summarizes a list of commonly used methods. + +| Method | Argument | Return | Description | +| -------------- | ----------- | ------ | ----------- | +| name | string | self | assigns a human-readable name to the task | +| work | callable | self | assigns a work of a callable object to the task | +| precede | task list | self | enables this task to run *before* the given tasks | +| succeed | task list | self | enables this task to run *after* the given tasks | +| num_dependents | none | size | returns the number of dependents (inputs) of this task | +| num_successors | none | size | returns the number of successors (outputs) of this task | +| empty | none | bool | returns true if the task points to a graph node or false otherwise | +| has_work | none | bool | returns true if the task points to a graph node with a callable assigned | + +### *name* + +The method `name` lets you assign a human-readable string to a task. + +```cpp +A.name("my name is A"); +``` + +### *work* + +The method `work` lets you assign a callable to a task. + +```cpp +A.work([] () { std::cout << "hello world!"; }); +``` + +### *precede* + +The method `precede` lets you add a preceding link from self to other tasks. + + + +```cpp +// A runs before B, C, D, and E +A.precede(B, C, D, E); +``` + +The method `succeed` is similar to `precede` but operates in the opposite direction. + +### *empty/has_work* + +A task is empty is it is not associated with any graph node. + +```cpp +tf::Task task; // assert(task.empty()); +``` + +A placeholder task is associated with a graph node but has no work assigned yet. + +``` +tf::Task task = taskflow.placeholder(); // assert(!task.has_work()); +``` + +## Executor API + +The class `tf::Executor` is used for execution of one or multiple taskflow objects. +The table below summarizes a list of commonly used methods. + +| Method | Argument | Return | Description | +| --------- | -------------- | ------------- | ------------------------ | +| run | taskflow | future | runs the taskflow once | +| run_n | taskflow, N | future | runs the taskflow N times | +| run_until | taskflow, binary predicate | future | keeps running the taskflow until the predicate becomes true | +| wait_for_all | none | none | blocks until all running tasks finish | +| make_observer | arguments to forward to user-derived constructor | pointer to the observer | creates an observer to monitor the thread activities of the executor | + +### *run/run_n/run_until* + +The run series are non-blocking call to execute a taskflow graph. +Issuing multiple runs on the same taskflow will automatically synchronize +to a sequential chain of executions. + +```cpp +executor.run(taskflow); // runs a graph once +executor.run_n(taskflow, 5); // runs a graph five times +executor.run_until(taskflow, my_pred); // keeps running until the my_pred becomes true +executor.wait_for_all(); // blocks until all tasks finish +``` + +The first run finishes before the second run, and the second run finishes before the third run. +
[↑]
+ +# System Requirements + +To use the latest [Cpp-Taskflow](https://github.com/cpp-taskflow/cpp-taskflow/archive/master.zip), you only need a [C++14][C++14] compiler. + ++ GNU C++ Compiler at least v5.0 with -std=c++14 ++ Clang C++ Compiler at least v4.0 with -std=c++14 ++ Microsoft Visual Studio at least v15.7 (MSVC++ 19.14); see [vcpkg guide](https://github.com/cpp-taskflow/cpp-taskflow/issues/143) ++ AppleClang Xode Version at least v8 ++ Nvidia CUDA Toolkit and Compiler ([nvcc][nvcc]) at least v10.0 with -std=c++14 + +Cpp-Taskflow works on Linux, Windows, and Mac OS X. See the [C++ compiler support](https://en.cppreference.com/w/cpp/compiler_support) status. + +
[↑]
+ +# Compile Unit Tests, Examples, and Benchmarks + +Cpp-Taskflow uses [CMake](https://cmake.org/) to build examples and unit tests. +We recommend using out-of-source build. + +```bash +~$ cmake --version # must be at least 3.9 or higher +~$ mkdir build +~$ cd build +~$ cmake ../ +~$ make & make test # run all unit tests +``` + +## Examples + +The folder `examples/` contains several examples and is a great place to learn to use Cpp-Taskflow. + +| Example | Description | +| ------- | ----------- | +| [simple.cpp](./examples/simple.cpp) | uses basic task building blocks to create a trivial taskflow graph | +| [debug.cpp](./examples/debug.cpp)| inspects a taskflow through the dump method | +| [parallel_for.cpp](./examples/parallel_for.cpp)| parallelizes a for loop with unbalanced workload | +| [subflow.cpp](./examples/subflow.cpp)| demonstrates how to create a subflow graph that spawns three dynamic tasks | +| [run_variants.cpp](./examples/run_variants.cpp)| shows multiple ways to run a taskflow graph | +| [composition.cpp](./examples/composition.cpp)| demonstrates the decomposable interface of taskflow | +| [observer.cpp](./examples/observer.cpp)| demonstrates how to monitor the thread activities in scheduling and running tasks | +| [condition.cpp](./examples/condition.cpp) | creates a conditional tasking graph with a feedback loop control flow | +| [cuda/saxpy.cu](./examples/cuda/saxpy.cu) | uses cudaFlow to create a saxpy (single-precision A·X Plus Y) task graph | +| [cuda/matmul.cu](./examples/cuda/matmul.cu) | uses cudaFlow to create a matrix multiplication workload and compares it with a CPU basline | + +## Benchmarks + +Please visit [benchmarks](benchmarks/benchmarks.md) to learn to +compile the benchmarks. + +
[↑]
+ +# Who is Using Cpp-Taskflow? + +Cpp-Taskflow is being used in both industry and academic projects to scale up existing workloads +that incorporate complex task dependencies. + +- [OpenTimer][OpenTimer]: A High-performance Timing Analysis Tool for Very Large Scale Integration (VLSI) Systems +- [DtCraft][DtCraft]: A General-purpose Distributed Programming Systems using Data-parallel Streams +- [Firestorm][Firestorm]: Fighting Game Engine with Asynchronous Resource Loaders (developed by [ForgeMistress][ForgeMistress]) +- [Shiva][Shiva]: An extensible engine via an entity component system through scripts, DLLs, and header-only (C++) +- [PID Framework][PID Framework]: A Global Development Methodology Supported by a CMake API and Dedicated C++ Projects +- [NovusCore][NovusCore]: An emulating project for World of Warraft (Wrath of the Lich King 3.3.5a 12340 client build) +- [SA-PCB][SA-PCB]: Annealing-based Printed Circuit Board (PCB) Placement Tool +- [LPMP](https://github.com/LPMP/LPMP): A C++ framework for developing scalable Lagrangian decomposition solvers for discrete optimization problems +- [Heteroflow](https://github.com/Heteroflow/Heteroflow): A Modern C++ Parallel CPU-GPU Task Programming Library +- [OpenPhySyn](https://github.com/The-OpenROAD-Project/OpenPhySyn): A plugin-based physical synthesis optimization kit as part of the OpenRoad flow +- [OSSIA](https://ossia.io/): Open-source Software System for Interactive Applications + +[More...](https://github.com/search?q=cpp-taskflow&type=Code) + +
[↑]
+ +# Contributors + +Cpp-Taskflow is being actively developed and contributed by the +[these people](https://github.com/cpp-taskflow/cpp-taskflow/graphs/contributors). +Meanwhile, we appreciate the support from many organizations for our developments. + + +| [][UofU] | [][UIUC] | [][CSL] | [][NSF] | [][DARPA IDEA] | +| :---: | :---: | :---: | :---: | :---: | + +# License + +Cpp-Taskflow is licensed under the [MIT License](./LICENSE). + +* * * + +[Tsung-Wei Huang]: https://tsung-wei-huang.github.io/ +[Chun-Xun Lin]: https://github.com/clin99 +[Martin Wong]: https://ece.illinois.edu/directory/profile/mdfwong +[Andreas Olofsson]: https://github.com/aolofsson +[Gitter]: https://gitter.im/cpp-taskflow/Lobby +[Gitter badge]: ./image/gitter_badge.svg +[GitHub releases]: https://github.com/coo-taskflow/cpp-taskflow/releases +[GitHub issues]: https://github.com/cpp-taskflow/cpp-taskflow/issues +[GitHub insights]: https://github.com/cpp-taskflow/cpp-taskflow/pulse +[GitHub pull requests]: https://github.com/cpp-taskflow/cpp-taskflow/pulls +[GitHub contributors]: https://github.com/cpp-taskflow/cpp-taskflow/graphs/contributors +[GraphViz]: https://www.graphviz.org/ +[AwesomeGraphViz]: https://dreampuf.github.io/GraphvizOnline/ +[OpenMP Tasking]: https://www.openmp.org/spec-html/5.0/openmpsu99.html +[TBB FlowGraph]: https://www.threadingbuildingblocks.org/tutorial-intel-tbb-flow-graph +[OpenTimer]: https://github.com/OpenTimer/OpenTimer +[DtCraft]: https://github.com/tsung-wei-huang/DtCraft +[totalgee]: https://github.com/totalgee +[damienhocking]: https://github.com/damienhocking +[ForgeMistress]: https://github.com/ForgeMistress +[Patrik Huber]: https://github.com/patrikhuber +[DARPA IDEA]: https://www.darpa.mil/news-events/2017-09-13 +[KingDuckZ]: https://github.com/KingDuckZ +[NSF]: https://www.nsf.gov/ +[UIUC]: https://illinois.edu/ +[CSL]: https://csl.illinois.edu/ +[UofU]: https://www.utah.edu/ +[wiki]: https://cpp-taskflow.github.io/cpp-taskflow/index.html +[release notes]: https://cpp-taskflow.github.io/cpp-taskflow/Releases.html +[PayMe]: https://www.paypal.me/twhuang/10 +[C++17]: https://en.wikipedia.org/wiki/C%2B%2B17 +[C++14]: https://en.wikipedia.org/wiki/C%2B%2B14 +[email me]: mailto:twh760812@gmail.com +[Cpp Conference 2018]: https://github.com/CppCon/CppCon2018 +[ChromeTracing]: https://www.chromium.org/developers/how-tos/trace-event-profiling-tool +[IPDPS19]: https://tsung-wei-huang.github.io/papers/ipdps19.pdf +[WorkStealing Wiki]: https://en.wikipedia.org/wiki/Work_stealing + +[std::invoke]: https://en.cppreference.com/w/cpp/utility/functional/invoke +[std::future]: https://en.cppreference.com/w/cpp/thread/future + +[cuda-zone]: https://developer.nvidia.com/cuda-zone +[nvcc]: https://developer.nvidia.com/cuda-llvm-compiler +[cuda-toolkit]: https://developer.nvidia.com/cuda-toolkit +[cudaGraph]: https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__GRAPH.html + +[Firestorm]: https://github.com/ForgeMistress/Firestorm +[Shiva]: https://shiva.gitbook.io/project/shiva +[PID Framework]: http://pid.lirmm.net/pid-framework/index.html +[NovusCore]: https://github.com/novuscore/NovusCore +[SA-PCB]: https://github.com/choltz95/SA-PCB + +[Presentation]: https://cpp-taskflow.github.io/ +[chrome://tracing]: chrome://tracing + diff --git a/bundled/cpp-taskflow/include/taskflow/core/error.hpp b/bundled/cpp-taskflow/include/taskflow/core/error.hpp new file mode 100644 index 0000000000..67391215c0 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/error.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include + +#include "../utility/stringify.hpp" + +namespace tf { + +// Procedure: throw_se +// Throws the system error under a given error code. +template +//void throw_se(const char* fname, const size_t line, Error::Code c, ArgsT&&... args) { +void throw_re(const char* fname, const size_t line, ArgsT&&... args) { + std::ostringstream oss; + oss << "[" << fname << ":" << line << "] "; + ostreamize(oss, std::forward(args)...); + //(oss << ... << args); + throw std::runtime_error(oss.str()); +} + +} // ------------------------------------------------------------------------ + +#define TF_THROW(...) tf::throw_re(__FILE__, __LINE__, __VA_ARGS__); + diff --git a/bundled/cpp-taskflow/include/taskflow/core/executor.hpp b/bundled/cpp-taskflow/include/taskflow/core/executor.hpp new file mode 100644 index 0000000000..8d178ee221 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/executor.hpp @@ -0,0 +1,1311 @@ +#pragma once + +#include "tsq.hpp" +#include "notifier.hpp" +#include "observer.hpp" +#include "taskflow.hpp" + +namespace tf { + + +/** @class WorkerView + +@brief class to access worker information from the observer interface + +*/ +//class WorkerView { +// +// friend class Executor; +// +// public: +// +// +// private: +// +// Worker* _worker; +// +//}; + + +// ---------------------------------------------------------------------------- +// Executor Definition +// ---------------------------------------------------------------------------- + + +/** @class Executor + +@brief execution interface for running a taskflow graph + +An executor object manages a set of worker threads and implements +an efficient work-stealing scheduling algorithm to run a taskflow. + +*/ +class Executor { + + struct Worker { + size_t id; + size_t victim; + Domain domain; + Executor* executor; + Notifier::Waiter* waiter; + std::mt19937 rdgen { std::random_device{}() }; + TaskQueue wsq[NUM_DOMAINS]; + Node* cache {nullptr}; + }; + + struct PerThread { + Worker* worker {nullptr}; + }; + +#ifdef TF_ENABLE_CUDA + struct cudaDevice { + std::vector streams; + }; +#endif + + public: + +#ifdef TF_ENABLE_CUDA + /** + @brief constructs the executor with N/M cpu/gpu worker threads + */ + explicit Executor( + size_t N = std::thread::hardware_concurrency(), + size_t M = cuda_num_devices() + ); +#else + /** + @brief constructs the executor with N worker threads + */ + explicit Executor(size_t N = std::thread::hardware_concurrency()); +#endif + + /** + @brief destructs the executor + */ + ~Executor(); + + /** + @brief runs the taskflow once + + @param taskflow a tf::Taskflow object + + @return a std::future to access the execution status of the taskflow + */ + std::future run(Taskflow& taskflow); + + /** + @brief runs the taskflow once and invoke a callback upon completion + + @param taskflow a tf::Taskflow object + @param callable a callable object to be invoked after this run + + @return a std::future to access the execution status of the taskflow + */ + template + std::future run(Taskflow& taskflow, C&& callable); + + /** + @brief runs the taskflow for N times + + @param taskflow a tf::Taskflow object + @param N number of runs + + @return a std::future to access the execution status of the taskflow + */ + std::future run_n(Taskflow& taskflow, size_t N); + + /** + @brief runs the taskflow for N times and then invokes a callback + + @param taskflow a tf::Taskflow + @param N number of runs + @param callable a callable object to be invoked after this run + + @return a std::future to access the execution status of the taskflow + */ + template + std::future run_n(Taskflow& taskflow, size_t N, C&& callable); + + /** + @brief runs the taskflow multiple times until the predicate becomes true and + then invokes a callback + + @param taskflow a tf::Taskflow + @param pred a boolean predicate to return true for stop + + @return a std::future to access the execution status of the taskflow + */ + template + std::future run_until(Taskflow& taskflow, P&& pred); + + /** + @brief runs the taskflow multiple times until the predicate becomes true and + then invokes the callback + + @param taskflow a tf::Taskflow + @param pred a boolean predicate to return true for stop + @param callable a callable object to be invoked after this run + + @return a std::future to access the execution status of the taskflow + */ + template + std::future run_until(Taskflow& taskflow, P&& pred, C&& callable); + + /** + @brief wait for all pending graphs to complete + */ + void wait_for_all(); + + /** + @brief queries the number of worker threads (can be zero) + */ + size_t num_workers() const; + + /** + @brief queries the number of running topologies at the time of this call + + When a taskflow is submitted to an executor, a topology is created to store + runtime metadata of the running taskflow. + */ + size_t num_topologies() const; + + /** + @brief queries the number of worker domains + + Each domain manages a subset of worker threads to execute domain-specific tasks, + for example, HOST tasks and CUDA tasks. + */ + size_t num_domains() const; + + /** + @brief queries the id of the caller thread in this executor + + Each worker has an unique id from 0 to N-1 exclusive to the associated executor. + If the caller thread does not belong to the executor, -1 is returned. + */ + int this_worker_id() const; + + /** + @brief constructs an observer to inspect the activities of worker threads + + Each executor manage a list of observers in shared ownership with callers. + + @tparam Observer observer type derived from tf::ObserverInterface + @tparam ArgsT... argument parameter pack + + @param args arguments to forward to the constructor of the observer + + @return a shared pointer to the created observer + */ + template + std::shared_ptr make_observer(Args&&... args); + + /** + @brief removes the associated observer + */ + template + void remove_observer(std::shared_ptr observer); + + /** + @brief queries the number of observers + */ + size_t num_observers() const; + + private: + + const size_t _VICTIM_BEG; + const size_t _VICTIM_END; + const size_t _MAX_STEALS; + const size_t _MAX_YIELDS; + + std::condition_variable _topology_cv; + std::mutex _topology_mutex; + std::mutex _wsq_mutex; + + size_t _num_topologies {0}; + + std::vector _workers; + std::vector _threads; + +#ifdef TF_ENABLE_CUDA + std::vector _cuda_devices; +#endif + + Notifier _notifier[NUM_DOMAINS]; + + TaskQueue _wsq[NUM_DOMAINS]; + + size_t _id_offset[NUM_DOMAINS] = {0}; + + std::atomic _num_actives[NUM_DOMAINS]; + std::atomic _num_thieves[NUM_DOMAINS]; + std::atomic _done {0}; + + std::unordered_set> _observers; + + TFProfObserver* _tfprof; + + PerThread& _per_thread() const; + + bool _wait_for_task(Worker&, Node*&); + + void _instantiate_tfprof(); + void _flush_tfprof(); + void _observer_prologue(Worker&, Node*); + void _observer_epilogue(Worker&, Node*); + void _spawn(size_t, Domain); + void _worker_loop(Worker&); + void _exploit_task(Worker&, Node*&); + void _explore_task(Worker&, Node*&); + void _schedule(Node*, bool); + void _schedule(PassiveVector&); + void _invoke(Worker&, Node*); + void _invoke_static_work(Worker&, Node*); + void _invoke_dynamic_work(Worker&, Node*, bool&); + void _invoke_condition_work(Worker&, Node*); + void _invoke_module_work(Worker&, Node*, bool&); + +#ifdef TF_ENABLE_CUDA + void _invoke_cudaflow_work(Worker&, Node*); + void _invoke_cudaflow_work_impl(Worker&, Node*); +#endif + + void _set_up_topology(Topology*); + void _tear_down_topology(Topology**); + void _increment_topology(); + void _decrement_topology(); + void _decrement_topology_and_notify(); +}; + + +#ifdef TF_ENABLE_CUDA +// Constructor +inline Executor::Executor(size_t N, size_t M) : + _VICTIM_BEG {0}, + _VICTIM_END {N + M - 1}, + _MAX_STEALS {(N + M + 1) << 1}, + _MAX_YIELDS {100}, + _workers {N + M}, + _cuda_devices {cuda_num_devices()}, + _notifier {Notifier(N), Notifier(M)} { + + if(N == 0) { + TF_THROW("no cpu workers to execute taskflows"); + } + + if(M == 0) { + TF_THROW("no gpu workers to execute cudaflows"); + } + + for(int i=0; i().get(); +} + +// Procedure: _flush_tfprof +inline void Executor::_flush_tfprof() { + if(_tfprof) { + std::ostringstream fpath; + fpath << get_env("TF_ENABLE_PROFILER") << _tfprof->_uuid << ".tfp"; + std::ofstream ofs(fpath.str()); + _tfprof->dump(ofs); + } +} + +// Function: num_workers +inline size_t Executor::num_workers() const { + return _workers.size(); +} + +// Function: num_domains +inline size_t Executor::num_domains() const { + return NUM_DOMAINS; +} + +// Function: num_topologies +inline size_t Executor::num_topologies() const { + return _num_topologies; +} + +// Function: _per_thread +inline Executor::PerThread& Executor::_per_thread() const { + thread_local PerThread pt; + return pt; +} + +// Function: this_worker_id +inline int Executor::this_worker_id() const { + auto worker = _per_thread().worker; + return worker ? static_cast(worker->id) : -1; +} + +// Procedure: _spawn +inline void Executor::_spawn(size_t N, Domain d) { + + auto id = _threads.size(); + + _id_offset[d] = id; + + for(size_t i=0; i void { + + PerThread& pt = _per_thread(); + pt.worker = &w; + + Node* t = nullptr; + + // must use 1 as condition instead of !done + while(1) { + + // execute the tasks. + _exploit_task(w, t); + + // wait for tasks + if(_wait_for_task(w, t) == false) { + break; + } + } + + }, std::ref(_workers[id])); + } + +} + +// Function: _explore_task +inline void Executor::_explore_task(Worker& w, Node*& t) { + + //assert(_workers[w].wsq.empty()); + assert(!t); + + const auto d = w.domain; + + size_t num_steals = 0; + size_t num_yields = 0; + + std::uniform_int_distribution rdvtm(_VICTIM_BEG, _VICTIM_END); + + //while(!_done) { + // + // size_t vtm = rdvtm(w.rdgen); + // + // t = (vtm == w.id) ? _wsq[d].steal() : _workers[vtm].wsq[d].steal(); + + // if(t) { + // break; + // } + + // if(num_steal++ > _MAX_STEALS) { + // std::this_thread::yield(); + // if(num_yields++ > _MAX_YIELDS) { + // break; + // } + // } + //} + + do { + t = (w.id == w.victim) ? _wsq[d].steal() : _workers[w.victim].wsq[d].steal(); + + if(t) { + break; + } + + if(num_steals++ > _MAX_STEALS) { + std::this_thread::yield(); + if(num_yields++ > _MAX_YIELDS) { + break; + } + } + + w.victim = rdvtm(w.rdgen); + } while(!_done); + +} + +// Procedure: _exploit_task +inline void Executor::_exploit_task(Worker& w, Node*& t) { + + assert(!w.cache); + + if(t) { + + const auto d = w.domain; + + if(_num_actives[d].fetch_add(1) == 0 && _num_thieves[d] == 0) { + _notifier[d].notify(false); + } + + auto tpg = t->_topology; + auto par = t->_parent; + auto exe = size_t{1}; + + do { + _invoke(w, t); + + if(w.cache) { + t = w.cache; + w.cache = nullptr; + } + else { + t = w.wsq[d].pop(); + if(t) { + // We only increment the counter when poping task from wsq + // (NOT including cache!) + if(t->_parent == par) { + exe++; + } + // joined subflow + else { + if(par == nullptr) { + // still have tasks so the topology join counter can't be zero + t->_topology->_join_counter.fetch_sub(exe); + } + else { + auto ret = par->_join_counter.fetch_sub(exe); + if(ret == exe) { + if(par->domain() == d) { + w.wsq[d].push(par); + } + else { + _schedule(par, false); + } + } + } + exe = 1; + par = t->_parent; + } + } + else { + // If no more local tasks! + if(par == nullptr) { + if(tpg->_join_counter.fetch_sub(exe) == exe) { + // TODO: Store tpg in local variable not in w + _tear_down_topology(&tpg); + if(tpg != nullptr) { + t = w.wsq[d].pop(); + if(t) { + exe = 1; + } + } + } + } + else { + if(par->_join_counter.fetch_sub(exe) == exe) { + if(par->domain() == d) { + t = par; + par = par->_parent; + exe = 1; + } + else { + _schedule(par, false); + } + } + } + } + } + } while(t); + + --_num_actives[d]; + } +} + +// Function: _wait_for_task +inline bool Executor::_wait_for_task(Worker& worker, Node*& t) { + + const auto d = worker.domain; + + wait_for_task: + + assert(!t); + + ++_num_thieves[d]; + + explore_task: + + _explore_task(worker, t); + + if(t) { + if(_num_thieves[d].fetch_sub(1) == 1) { + _notifier[d].notify(false); + } + return true; + } + + _notifier[d].prepare_wait(worker.waiter); + + //if(auto vtm = _find_victim(me); vtm != _workers.size()) { + if(!_wsq[d].empty()) { + + _notifier[d].cancel_wait(worker.waiter); + //t = (vtm == me) ? _wsq.steal() : _workers[vtm].wsq.steal(); + + t = _wsq[d].steal(); + if(t) { + if(_num_thieves[d].fetch_sub(1) == 1) { + _notifier[d].notify(false); + } + return true; + } + else { + worker.victim = worker.id; + goto explore_task; + } + } + + if(_done) { + _notifier[d].cancel_wait(worker.waiter); + for(int i=0; i +std::shared_ptr Executor::make_observer(Args&&... args) { + + static_assert( + std::is_base_of::value, + "Observer must be derived from ObserverInterface" + ); + + // use a local variable to mimic the constructor + auto ptr = std::make_shared(std::forward(args)...); + + ptr->set_up(_workers.size()); + + _observers.emplace(std::static_pointer_cast(ptr)); + + return ptr; +} + +// Procedure: remove_observer +template +void Executor::remove_observer(std::shared_ptr ptr) { + + static_assert( + std::is_base_of::value, + "Observer must be derived from ObserverInterface" + ); + + _observers.erase(std::static_pointer_cast(ptr)); +} + +// Function: num_observers +inline size_t Executor::num_observers() const { + return _observers.size(); +} + +// Procedure: _schedule +// The main procedure to schedule a give task node. +// Each task node has two types of tasks - regular and subflow. +inline void Executor::_schedule(Node* node, bool bypass_hint) { + + //assert(_workers.size() != 0); + + const auto d = node->domain(); + + // caller is a worker to this pool + auto worker = _per_thread().worker; + + if(worker != nullptr && worker->executor == this) { + if(bypass_hint) { + assert(!worker->cache); + worker->cache = node; + } + else { + worker->wsq[d].push(node); + if(worker->domain != d) { + if(_num_actives[d] == 0 && _num_thieves[d] == 0) { + _notifier[d].notify(false); + } + } + } + return; + } + + // other threads + { + std::lock_guard lock(_wsq_mutex); + _wsq[d].push(node); + } + + _notifier[d].notify(false); +} + +// Procedure: _schedule +// The main procedure to schedule a set of task nodes. +// Each task node has two types of tasks - regular and subflow. +inline void Executor::_schedule(PassiveVector& nodes) { + + //assert(_workers.size() != 0); + + // We need to cacth the node count to avoid accessing the nodes + // vector while the parent topology is removed! + const auto num_nodes = nodes.size(); + + if(num_nodes == 0) { + return; + } + + // worker thread + auto worker = _per_thread().worker; + + // task counts + size_t tcount[NUM_DOMAINS] = {0}; + + if(worker != nullptr && worker->executor == this) { + for(size_t i=0; idomain(); + worker->wsq[d].push(nodes[i]); + tcount[d]++; + } + + for(int d=0; ddomain) { + if(_num_actives[d] == 0 && _num_thieves[d] == 0) { + _notifier[d].notify_n(tcount[d]); + } + } + } + + return; + } + + // other threads + { + std::lock_guard lock(_wsq_mutex); + for(size_t k=0; kdomain(); + _wsq[d].push(nodes[k]); + tcount[d]++; + } + } + + for(int d=0; dnum_successors(); + + // acquire the parent flow counter + auto& c = (node->_parent) ? node->_parent->_join_counter : + node->_topology->_join_counter; + + // switch is faster than nested if-else due to jump table + switch(node->_handle.index()) { + // static task + case Node::STATIC_WORK:{ + _invoke_static_work(worker, node); + } + break; + + // module task + case Node::MODULE_WORK: { + bool first_time = !node->_has_state(Node::SPAWNED); + bool emptiness = false; + _invoke_module_work(worker, node, emptiness); + if(first_time && !emptiness) { + return; + } + } + break; + + // dynamic task + case Node::DYNAMIC_WORK: { + // Need to create a subflow if it is the first time entering here + if(!node->_has_state(Node::SPAWNED)) { + bool join = false; + _invoke_dynamic_work(worker, node, join); + if(join) { + return; + } + } + } + break; + + // condition task + case Node::CONDITION_WORK: { + _invoke_condition_work(worker, node); + return ; + } // no need to add a break here due to the immediate return + + // cudaflow task +#ifdef TF_ENABLE_CUDA + case Node::CUDAFLOW_WORK: { + _invoke_cudaflow_work(worker, node); + } + break; +#endif + + // monostate + default: + break; + } + + + // We MUST recover the dependency since subflow may have + // a condition node to go back (cyclic). + // This must be done before scheduling the successors, otherwise this might cause + // race condition on the _dependents + if(node->_has_state(Node::BRANCH)) { + // If this is a case node, we need to deduct condition predecessors + node->_join_counter = node->num_strong_dependents(); + } + else { + node->_join_counter = node->num_dependents(); + } + + node->_unset_state(Node::SPAWNED); + + // At this point, the node storage might be destructed. + Node* cache {nullptr}; + + for(size_t i=0; i_successors[i]->_join_counter) == 0) { + if(node->_successors[i]->domain() != worker.domain) { + c.fetch_add(1); + _schedule(node->_successors[i], false); + } + else { + if(cache) { + c.fetch_add(1); + _schedule(cache, false); + } + cache = node->_successors[i]; + } + } + } + + if(cache) { + _schedule(cache, true); + } +} + +// Procedure: _observer_prologue +inline void Executor::_observer_prologue(Worker& worker, Node* node) { + for(auto& observer : _observers) { + observer->on_entry(worker.id, TaskView(node)); + } +} + +// Procedure: _observer_epilogue +inline void Executor::_observer_epilogue(Worker& worker, Node* node) { + for(auto& observer : _observers) { + observer->on_exit(worker.id, TaskView(node)); + } +} + +// Procedure: _invoke_static_work +inline void Executor::_invoke_static_work(Worker& worker, Node* node) { + _observer_prologue(worker, node); + nstd::get(node->_handle).work(); + _observer_epilogue(worker, node); +} + +// Procedure: _invoke_dynamic_work +inline void Executor::_invoke_dynamic_work(Worker& worker, Node* node, bool& join) { + + _observer_prologue(worker, node); + + auto& subgraph = nstd::get(node->_handle).subgraph; + + subgraph.clear(); + Subflow fb(subgraph); + + nstd::get(node->_handle).work(fb); + + node->_set_state(Node::SPAWNED); + + if(!subgraph.empty()) { + + PassiveVector src; + + for(auto n : subgraph._nodes) { + + n->_topology = node->_topology; + n->_set_up_join_counter(); + + if(!fb.detached()) { + n->_parent = node; + } + + if(n->num_dependents() == 0) { + src.push_back(n); + } + } + + join = fb.joined(); + + if(!join) { // Detach mode + node->_topology->_join_counter.fetch_add(src.size()); + } + else { // Join mode (spawned nodes need second-round execution + node->_join_counter.fetch_add(src.size()); + + node->_parent ? node->_parent->_join_counter.fetch_add(1) : + node->_topology->_join_counter.fetch_add(1); + } + + _schedule(src); + } + + _observer_epilogue(worker, node); +} + +// Procedure: _invoke_condition_work +inline void Executor::_invoke_condition_work(Worker& worker, Node* node) { + + _observer_prologue(worker, node); + + if(node->_has_state(Node::BRANCH)) { + node->_join_counter = node->num_strong_dependents(); + } + else { + node->_join_counter = node->num_dependents(); + } + + auto id = nstd::get(node->_handle).work(); + + if(id >= 0 && static_cast(id) < node->num_successors()) { + auto s = node->_successors[id]; + s->_join_counter.store(0); + + if(s->domain() == worker.domain) { + _schedule(s, true); + } + else { + node->_parent ? node->_parent->_join_counter.fetch_add(1) : + node->_topology->_join_counter.fetch_add(1); + _schedule(s, false); + } + } + + _observer_epilogue(worker, node); +} + +#ifdef TF_ENABLE_CUDA +// Procedure: _invoke_cudaflow_work +inline void Executor::_invoke_cudaflow_work(Worker& worker, Node* node) { + _observer_prologue(worker, node); + _invoke_cudaflow_work_impl(worker, node); + _observer_epilogue(worker, node); +} + +// Procedure: _invoke_cudaflow_work_impl +inline void Executor::_invoke_cudaflow_work_impl(Worker& w, Node* node) { + + assert(w.domain == node->domain()); + + auto& h = nstd::get(node->_handle); + + h.graph.clear(); + + cudaFlow cf(h.graph, [repeat=1] () mutable { return repeat-- == 0; }); + + h.work(cf); + + if(h.graph.empty()) { + return; + } + + // transforms cudaFlow to a native cudaGraph under the specified device + // and launches the graph through a given or an internal device stream + const int d = cf._device; + + cudaScopedDevice ctx(d); + + auto s = cf._stream ? *(cf._stream) : + _cuda_devices[d].streams[w.id - _id_offset[w.domain]]; + + h.graph._make_native_graph(); + + cudaGraphExec_t exec; + + TF_CHECK_CUDA( + cudaGraphInstantiate(&exec, h.graph._native_handle, nullptr, nullptr, 0), + "failed to create an executable cudaGraph" + ); + + while(!cf._predicate()) { + TF_CHECK_CUDA( + cudaGraphLaunch(exec, s), "failed to launch cudaGraph on stream ", s + ); + + TF_CHECK_CUDA( + cudaStreamSynchronize(s), "failed to synchronize stream ", s + ); + } + + TF_CHECK_CUDA( + cudaGraphExecDestroy(exec), "failed to destroy an executable cudaGraph" + ); +} +#endif + +// Procedure: _invoke_module_work +inline void Executor::_invoke_module_work(Worker& worker, Node* node, bool& ept) { + + // second time to enter this context + if(node->_has_state(Node::SPAWNED)) { + return; + } + + _observer_prologue(worker, node); + + // first time to enter this context + node->_set_state(Node::SPAWNED); + + auto module = nstd::get(node->_handle).module; + + if(module->empty()) { + ept = true; + return; + } + + PassiveVector src; + + for(auto n: module->_graph._nodes) { + + n->_topology = node->_topology; + n->_parent = node; + n->_set_up_join_counter(); + + if(n->num_dependents() == 0) { + src.push_back(n); + } + } + + node->_join_counter.fetch_add(src.size()); + + if(node->_parent == nullptr) { + node->_topology->_join_counter.fetch_add(1); + } + else { + node->_parent->_join_counter.fetch_add(1); + } + + // src can't be empty (banned outside) + _schedule(src); + + _observer_epilogue(worker, node); +} + +// Function: run +inline std::future Executor::run(Taskflow& f) { + return run_n(f, 1, [](){}); +} + +// Function: run +template +std::future Executor::run(Taskflow& f, C&& c) { + return run_n(f, 1, std::forward(c)); +} + +// Function: run_n +inline std::future Executor::run_n(Taskflow& f, size_t repeat) { + return run_n(f, repeat, [](){}); +} + +// Function: run_n +template +std::future Executor::run_n(Taskflow& f, size_t repeat, C&& c) { + return run_until(f, [repeat]() mutable { return repeat-- == 0; }, std::forward(c)); +} + +// Function: run_until +template +std::future Executor::run_until(Taskflow& f, P&& pred) { + return run_until(f, std::forward

(pred), [](){}); +} + +// Function: _set_up_topology +inline void Executor::_set_up_topology(Topology* tpg) { + + tpg->_sources.clear(); + + // scan each node in the graph and build up the links + for(auto node : tpg->_taskflow._graph._nodes) { + + node->_topology = tpg; + node->_clear_state(); + + if(node->num_dependents() == 0) { + tpg->_sources.push_back(node); + } + + node->_set_up_join_counter(); + } + + tpg->_join_counter.store(tpg->_sources.size(), std::memory_order_relaxed); +} + +// Function: _tear_down_topology +inline void Executor::_tear_down_topology(Topology** tpg) { + + auto &f = (*tpg)->_taskflow; + + //assert(&tpg == &(f._topologies.front())); + + // case 1: we still need to run the topology again + if(! (*tpg)->_pred() ) { + //tpg->_recover_num_sinks(); + + assert((*tpg)->_join_counter == 0); + (*tpg)->_join_counter = (*tpg)->_sources.size(); + + _schedule((*tpg)->_sources); + } + // case 2: the final run of this topology + else { + + if((*tpg)->_call != nullptr) { + (*tpg)->_call(); + } + + f._mtx.lock(); + + // If there is another run (interleave between lock) + if(f._topologies.size() > 1) { + + assert((*tpg)->_join_counter == 0); + + // Set the promise + (*tpg)->_promise.set_value(); + f._topologies.pop_front(); + f._mtx.unlock(); + + // decrement the topology but since this is not the last we don't notify + _decrement_topology(); + + *tpg = &(f._topologies.front()); + + _set_up_topology(*tpg); + _schedule((*tpg)->_sources); + + //f._topologies.front()._bind(f._graph); + //*tpg = &(f._topologies.front()); + + //assert(f._topologies.front()._join_counter == 0); + + //f._topologies.front()._join_counter = f._topologies.front()._sources.size(); + + //_schedule(f._topologies.front()._sources); + } + else { + assert(f._topologies.size() == 1); + + // Need to back up the promise first here becuz taskflow might be + // destroy before taskflow leaves + auto p {std::move((*tpg)->_promise)}; + + f._topologies.pop_front(); + + f._mtx.unlock(); + + // We set the promise in the end in case taskflow leaves before taskflow + p.set_value(); + + _decrement_topology_and_notify(); + + // Reset topology so caller can stop execution + *tpg = nullptr; + } + } +} + +// Function: run_until +template +std::future Executor::run_until(Taskflow& f, P&& pred, C&& c) { + + _increment_topology(); + + // Special case of predicate + if(f.empty() || pred()) { + std::promise promise; + promise.set_value(); + _decrement_topology_and_notify(); + return promise.get_future(); + } + + // Multi-threaded execution. + bool run_now {false}; + Topology* tpg; + std::future future; + + { + std::lock_guard lock(f._mtx); + + // create a topology for this run + //tpg = &(f._topologies.emplace_back(f, std::forward

(pred), std::forward(c))); + f._topologies.emplace_back(f, std::forward

(pred), std::forward(c)); + tpg = &(f._topologies.back()); + future = tpg->_promise.get_future(); + + if(f._topologies.size() == 1) { + run_now = true; + //tpg->_bind(f._graph); + //_schedule(tpg->_sources); + } + } + + // Notice here calling schedule may cause the topology to be removed sonner + // before the function leaves. + if(run_now) { + _set_up_topology(tpg); + _schedule(tpg->_sources); + } + + return future; +} + +// Procedure: _increment_topology +inline void Executor::_increment_topology() { + std::lock_guard lock(_topology_mutex); + ++_num_topologies; +} + +// Procedure: _decrement_topology_and_notify +inline void Executor::_decrement_topology_and_notify() { + std::lock_guard lock(_topology_mutex); + if(--_num_topologies == 0) { + _topology_cv.notify_all(); + } +} + +// Procedure: _decrement_topology +inline void Executor::_decrement_topology() { + std::lock_guard lock(_topology_mutex); + --_num_topologies; +} + +// Procedure: wait_for_all +inline void Executor::wait_for_all() { + std::unique_lock lock(_topology_mutex); + _topology_cv.wait(lock, [&](){ return _num_topologies == 0; }); +} + +} // end of namespace tf ----------------------------------------------------- + + diff --git a/bundled/cpp-taskflow/include/taskflow/core/flow_builder.hpp b/bundled/cpp-taskflow/include/taskflow/core/flow_builder.hpp new file mode 100644 index 0000000000..d0677dbf20 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/flow_builder.hpp @@ -0,0 +1,930 @@ +#pragma once + +#include "task.hpp" + +namespace tf { + +/** +@class FlowBuilder + +@brief building methods of a task dependency graph + +*/ +class FlowBuilder { + + friend class Task; + + public: + + /** + @brief constructs a flow builder object + + @param graph a task dependency graph to manipulate + */ + FlowBuilder(Graph& graph); + + /** + @brief creates a static task from a given callable object + + @tparam C callable type + + @param callable a callable object constructible from std::function + + @return Task handle + */ + template + std::enable_if_t, Task> emplace(C&& callable); + + /** + @brief creates a dynamic task from a given callable object + + @tparam C callable type + + @param callable a callable object constructible from std::function + + @return Task handle + */ + template + std::enable_if_t, Task> emplace(C&& callable); + + /** + @brief creates a condition task from a given callable object + + @tparam C callable type + + @param callable a callable object constructible from std::function + + @return Task handle + */ + template + std::enable_if_t, Task> emplace(C&& callable); + +#ifdef TF_ENABLE_CUDA + /** + @brief creates a cudaflow task from a given callable object + + @tparam C callable type + + @param callable a callable object constructible from std::function + + @return Task handle + */ + template + std::enable_if_t, Task> emplace(C&& callable); +#endif + + /** + @brief creates multiple tasks from a list of callable objects + + @tparam C... callable types + + @param callables one or multiple callable objects constructible from each task category + + @return a Task handle + */ + template 1), void>* = nullptr> + auto emplace(C&&... callables); + + /** + @brief creates a module task from a taskflow + + @param taskflow a taskflow object for the module + @return a Task handle + */ + Task composed_of(Taskflow& taskflow); + + /** + @brief constructs a task dependency graph of range-based parallel_for + + The task dependency graph applies a callable object + to the dereferencing of every iterator + in the range [beg, end) chunk by chunk. + + @tparam I input iterator type + @tparam C callable type + + @param beg iterator to the beginning (inclusive) + @param end iterator to the end (exclusive) + @param callable a callable object to be applied to + @param chunk size (default 1) + + @return a pair of Task handles to the beginning and the end of the graph + */ + template + std::pair parallel_for(I beg, I end, C&& callable, size_t chunk=1); + + /** + @brief constructs a task dependency graph of integer index-based parallel_for + + The task dependency graph applies a callable object to every index + in the range [beg, end) with a step size chunk by chunk. + + @tparam I integer (arithmetic) index type + @tparam C callable type + + @param beg index of the beginning (inclusive) + @param end index of the end (exclusive) + @param step step size + @param callable a callable object to be applied to + @param chunk items per task + + @return a pair of Task handles to the beginning and the end of the graph + */ + template < + typename I, + typename C, + std::enable_if_t>::value, void>* = nullptr + > + std::pair parallel_for( + I beg, I end, I step, C&& callable, size_t chunk = 1 + ); + + /** + @brief constructs a task dependency graph of floating index-based parallel_for + + The task dependency graph applies a callable object to every index + in the range [beg, end) with a step size chunk by chunk. + + @tparam I floating (arithmetic) index type + @tparam C callable type + + @param beg index of the beginning (inclusive) + @param end index of the end (exclusive) + @param step step size + @param callable a callable object to be applied to + @param chunk items per task + + @return a pair of Task handles to the beginning and the end of the graph + */ + template < + typename I, + typename C, + std::enable_if_t>::value, void>* = nullptr + > + std::pair parallel_for( + I beg, I end, I step, C&& callable, size_t chunk = 1 + ); + + /** + @brief construct a task dependency graph of parallel reduction + + The task dependency graph reduces items in the range [beg, end) to a single result. + + @tparam I input iterator type + @tparam T data type + @tparam B binary operator type + + @param beg iterator to the beginning (inclusive) + @param end iterator to the end (exclusive) + @param result reference variable to store the final result + @param bop binary operator that will be applied in unspecified order to the result + of dereferencing the input iterator + + @return a pair of Task handles to the beginning and the end of the graph + */ + template + std::pair reduce(I beg, I end, T& result, B&& bop); + + /** + @brief constructs a task dependency graph of parallel reduction through @std_min + + The task dependency graph applies a parallel reduction + to find the minimum item in the range [beg, end) through @std_min reduction. + + @tparam I input iterator type + @tparam T data type + + @param beg iterator to the beginning (inclusive) + @param end iterator to the end (exclusive) + @param result reference variable to store the final result + + @return a pair of Task handles to the beginning and the end of the graph + */ + template + std::pair reduce_min(I beg, I end, T& result); + + /** + @brief constructs a task dependency graph of parallel reduction through @std_max + + The task dependency graph applies a parallel reduction + to find the maximum item in the range [beg, end) through @std_max reduction. + + @tparam I input iterator type + @tparam T data type + + @param beg iterator to the beginning (inclusive) + @param end iterator to the end (exclusive) + @param result reference variable to store the final result + + @return a pair of Task handles to the beginning and the end of the graph + */ + template + std::pair reduce_max(I beg, I end, T& result); + + /** + @brief constructs a task dependency graph of parallel transformation and reduction + + The task dependency graph transforms each item in the range [beg, end) + into a new data type and then reduce the results. + + @tparam I input iterator type + @tparam T data type + @tparam B binary operator + @tparam U unary operator type + + @param beg iterator to the beginning (inclusive) + @param end iterator to the end (exclusive) + @param result reference variable to store the final result + @param bop binary function object that will be applied in unspecified order + to the results of @c uop; the return type must be @c T + @param uop unary function object that transforms each element + in the input range; the return type must be acceptable as input to @c bop + + @return a pair of Task handles to the beginning and the end of the graph + */ + template + std::pair transform_reduce(I beg, I end, T& result, B&& bop, U&& uop); + + /** + @brief constructs a task dependency graph of parallel transformation and reduction + + The task dependency graph transforms each item in the range [beg, end) + into a new data type and then apply two-layer reductions to derive the result. + + @tparam I input iterator type + @tparam T data type + @tparam B binary operator type + @tparam P binary operator type + @tparam U unary operator type + + @param beg iterator to the beginning (inclusive) + @param end iterator to the end (exclusive) + @param result reference variable to store the final result + @param bop1 binary function object that will be applied in the second-layer reduction + to the results of @c bop2 + @param bop2 binary function object that will be applied in the first-layer reduction + to the results of @c uop and the dereferencing of input iterators + @param uop unary function object that will be applied to transform an item to a new + data type that is acceptable as input to @c bop2 + + @return a pair of Task handles to the beginning and the end of the graph + */ + template + std::pair transform_reduce( + I beg, I end, T& result, B&& bop1, P&& bop2, U&& uop + ); + + /** + @brief creates an empty task + + @return a Task handle + */ + Task placeholder(); + + /** + @brief adds a dependency link from task A to task B + + @param A task A + @param B task B + */ + void precede(Task A, Task B); + + /** + @brief adds adjacent dependency links to a linear list of tasks + + @param tasks a vector of tasks + */ + void linearize(std::vector& tasks); + + /** + @brief adds adjacent dependency links to a linear list of tasks + + @param tasks an initializer list of tasks + */ + void linearize(std::initializer_list tasks); + + /** + @brief adds dependency links from one task A to many tasks + + @param A task A + @param others a task set which A precedes + */ + void broadcast(Task A, std::vector& others); + + /** + @brief adds dependency links from one task A to many tasks + + @param A task A + @param others a task set which A precedes + */ + void broadcast(Task A, std::initializer_list others); + + /** + @brief adds dependency links from many tasks to one task A + + @param others a task set to precede A + @param A task A + */ + void succeed(std::vector& others, Task A); + + /** + @brief adds dependency links from many tasks to one task A + + @param others a task set to precede A + @param A task A + */ + void succeed(std::initializer_list others, Task A); + + private: + + Graph& _graph; + + template + void _linearize(L&); +}; + +// Constructor +inline FlowBuilder::FlowBuilder(Graph& graph) : + _graph {graph} { +} + +// Function: emplace +template 1), void>*> +auto FlowBuilder::emplace(C&&... cs) { + return std::make_tuple(emplace(std::forward(cs))...); +} + +// Function: emplace +// emplaces a static task +template +std::enable_if_t, Task> FlowBuilder::emplace(C&& c) { + auto n = _graph.emplace_back( + nstd::in_place_type_t{}, std::forward(c) + ); + return Task(n); +} + +// Function: emplace +// emplaces a dynamic task +template +std::enable_if_t, Task> FlowBuilder::emplace(C&& c) { + auto n = _graph.emplace_back( + nstd::in_place_type_t{}, std::forward(c) + ); + return Task(n); +} + +// Function: emplace +// emplaces a condition task +template +std::enable_if_t, Task> FlowBuilder::emplace(C&& c) { + auto n = _graph.emplace_back( + nstd::in_place_type_t{}, std::forward(c) + ); + return Task(n); +} + +#ifdef TF_ENABLE_CUDA +// Function: emplace +// emplaces a cudaflow task +template +std::enable_if_t, Task> FlowBuilder::emplace(C&& c) { + auto n = _graph.emplace_back( + nstd::in_place_type_t{}, std::forward(c) + ); + return Task(n); +} +#endif + +// Function: composed_of +inline Task FlowBuilder::composed_of(Taskflow& taskflow) { + auto node = _graph.emplace_back( + nstd::in_place_type_t{}, &taskflow + ); + return Task(node); +} + +// Procedure: precede +inline void FlowBuilder::precede(Task from, Task to) { + from._node->_precede(to._node); +} + +// Procedure: broadcast +inline void FlowBuilder::broadcast(Task from, std::vector& tos) { + for(auto to : tos) { + from.precede(to); + } +} + +// Procedure: broadcast +inline void FlowBuilder::broadcast(Task from, std::initializer_list tos) { + for(auto to : tos) { + from.precede(to); + } +} + +// Function: succeed +inline void FlowBuilder::succeed(std::vector& froms, Task to) { + for(auto from : froms) { + to.succeed(from); + } +} + +// Function: succeed +inline void FlowBuilder::succeed(std::initializer_list froms, Task to) { + for(auto from : froms) { + to.succeed(from); + } +} + +// Function: placeholder +inline Task FlowBuilder::placeholder() { + auto node = _graph.emplace_back(); + return Task(node); +} + +// Function: parallel_for +template +std::pair FlowBuilder::parallel_for( + I beg, I end, C&& c, size_t chunk +){ + + //using category = typename std::iterator_traits::iterator_category; + + auto S = placeholder(); + auto T = placeholder(); + + // default partition equals to the worker count + if(chunk == 0) { + chunk = 1; + } + + size_t remain = std::distance(beg, end); + + while(beg != end) { + + auto e = beg; + + auto x = std::min(remain, chunk); + std::advance(e, x); + remain -= x; + + // Create a task + auto task = emplace([beg, e, c] () mutable { + std::for_each(beg, e, c); + }); + + S.precede(task); + task.precede(T); + + // adjust the pointer + beg = e; + } + + // special case + if(S.num_successors() == 0) { + S.precede(T); + } + + return std::make_pair(S, T); +} + +// Function: parallel_for +template < + typename I, + typename C, + std::enable_if_t>::value, void>* +> +std::pair FlowBuilder::parallel_for(I beg, I end, I s, C&& c, size_t chunk) { + + if((s == 0) || (beg < end && s <= 0) || (beg > end && s >=0) ) { + TF_THROW("invalid range [", beg, ", ", end, ") with step size ", s); + } + + // source and target + auto source = placeholder(); + auto target = placeholder(); + + if(chunk == 0) { + chunk = 1; + } + + // positive case + if(beg < end) { + while(beg != end) { + auto o = static_cast(chunk) * s; + auto e = std::min(beg + o, end); + auto task = emplace([=] () mutable { + for(auto i=beg; i end) { + while(beg != end) { + auto o = static_cast(chunk) * s; + auto e = std::max(beg + o, end); + auto task = emplace([=] () mutable { + for(auto i=beg; i>e; i+=s) { + c(i); + } + }); + source.precede(task); + task.precede(target); + beg = e; + } + } + + if(source.num_successors() == 0) { + source.precede(target); + } + + return std::make_pair(source, target); +} + +// Function: parallel_for +template >::value, void>* +> +std::pair FlowBuilder::parallel_for(I beg, I end, I s, C&& c, size_t chunk) { + + if((s == 0) || (beg < end && s <= 0) || (beg > end && s >=0) ) { + TF_THROW("invalid range [", beg, ", ", end, ") with step size ", s); + } + + // source and target + auto source = placeholder(); + auto target = placeholder(); + + if(chunk == 0) { + chunk = 1; + } + + // positive case + if(beg < end) { + size_t N=0; + I b = beg; + for(I e=beg; e end) { + size_t N=0; + I b = beg; + for(I e=beg; e>end; e+=s) { + if(++N == chunk) { + auto task = emplace([=] () mutable { + for(size_t i=0; i +std::pair FlowBuilder::reduce_min(I beg, I end, T& result) { + return reduce(beg, end, result, [] (const auto& l, const auto& r) { + return std::min(l, r); + }); +} + +// Function: reduce_max +// Find the maximum element over a range of items. +template +std::pair FlowBuilder::reduce_max(I beg, I end, T& result) { + return reduce(beg, end, result, [] (const auto& l, const auto& r) { + return std::max(l, r); + }); +} + +// Function: transform_reduce +template +std::pair FlowBuilder::transform_reduce( + I beg, I end, T& result, B&& bop, U&& uop +) { + + //using category = typename std::iterator_traits::iterator_category; + + // Even partition + size_t d = std::distance(beg, end); + size_t w = std::max(unsigned{1}, std::thread::hardware_concurrency()); + size_t g = std::max((d + w - 1) / w, size_t{2}); + + auto source = placeholder(); + auto target = placeholder(); + + //std::vector> futures; + auto g_results = std::make_unique(w); + size_t id {0}; + + size_t remain = d; + + while(beg != end) { + + auto e = beg; + + size_t x = std::min(remain, g); + std::advance(e, x); + remain -= x; + + // Create a task + auto task = emplace([beg, e, bop, uop, res=&(g_results[id])] () mutable { + *res = uop(*beg); + for(++beg; beg != e; ++beg) { + *res = bop(std::move(*res), uop(*beg)); + } + }); + + source.precede(task); + task.precede(target); + + // adjust the pointer + beg = e; + id ++; + } + + // target synchronizer + target.work([&result, bop, res=make_moc(std::move(g_results)), w=id] () { + for(auto i=0u; i +std::pair FlowBuilder::transform_reduce( + I beg, I end, T& result, B&& bop, P&& pop, U&& uop +) { + + //using category = typename std::iterator_traits::iterator_category; + + // Even partition + size_t d = std::distance(beg, end); + size_t w = std::max(unsigned{1}, std::thread::hardware_concurrency()); + size_t g = std::max((d + w - 1) / w, size_t{2}); + + auto source = placeholder(); + auto target = placeholder(); + + auto g_results = std::make_unique(w); + + size_t id {0}; + size_t remain = d; + + while(beg != end) { + + auto e = beg; + + size_t x = std::min(remain, g); + std::advance(e, x); + remain -= x; + + // Create a task + auto task = emplace([beg, e, uop, pop, res= &g_results[id]] () mutable { + *res = uop(*beg); + for(++beg; beg != e; ++beg) { + *res = pop(std::move(*res), *beg); + } + }); + source.precede(task); + task.precede(target); + + // adjust the pointer + beg = e; + id ++; + } + + // target synchronizer + target.work([&result, bop, g_results=make_moc(std::move(g_results)), w=id] () { + for(auto i=0u; i +void FlowBuilder::_linearize(L& keys) { + + auto itr = keys.begin(); + auto end = keys.end(); + + if(itr == end) { + return; + } + + auto nxt = itr; + + for(++nxt; nxt != end; ++nxt, ++itr) { + itr->_node->_precede(nxt->_node); + } +} + +// Procedure: linearize +inline void FlowBuilder::linearize(std::vector& keys) { + _linearize(keys); +} + +// Procedure: linearize +inline void FlowBuilder::linearize(std::initializer_list keys) { + _linearize(keys); +} + +// Proceduer: reduce +template +std::pair FlowBuilder::reduce(I beg, I end, T& result, B&& op) { + + //using category = typename std::iterator_traits::iterator_category; + + size_t d = std::distance(beg, end); + size_t w = std::max(unsigned{1}, std::thread::hardware_concurrency()); + size_t g = std::max((d + w - 1) / w, size_t{2}); + + auto source = placeholder(); + auto target = placeholder(); + + //T* g_results = static_cast(malloc(sizeof(T)*w)); + auto g_results = std::make_unique(w); + //std::vector> futures; + + size_t id {0}; + size_t remain = d; + + while(beg != end) { + + auto e = beg; + + size_t x = std::min(remain, g); + std::advance(e, x); + remain -= x; + + // Create a task + //auto [task, future] = emplace([beg, e, op] () mutable { + auto task = emplace([beg, e, op, res = &g_results[id]] () mutable { + *res = *beg; + for(++beg; beg != e; ++beg) { + *res = op(std::move(*res), *beg); + } + //auto init = *beg; + //for(++beg; beg != e; ++beg) { + // init = op(std::move(init), *beg); + //} + //return init; + }); + source.precede(task); + task.precede(target); + //futures.push_back(std::move(future)); + + // adjust the pointer + beg = e; + id ++; + } + + // target synchronizer + //target.work([&result, futures=MoC{std::move(futures)}, op] () { + // for(auto& fu : futures.object) { + // result = op(std::move(result), fu.get()); + // } + //}); + target.work([g_results=make_moc(std::move(g_results)), &result, op, w=id] () { + for(auto i=0u; i + Subflow(Args&&... args); + + /** + @brief enables the subflow to join its parent task + */ + void join(); + + /** + @brief enables the subflow to detach from its parent task + */ + void detach(); + + /** + @brief queries if the subflow will be detached from its parent task + */ + bool detached() const; + + /** + @brief queries if the subflow will join its parent task + */ + bool joined() const; + + private: + + bool _detached {false}; +}; + +// Constructor +template +Subflow::Subflow(Args&&... args) : + FlowBuilder {std::forward(args)...} { +} + +// Procedure: join +inline void Subflow::join() { + _detached = false; +} + +// Procedure: detach +inline void Subflow::detach() { + _detached = true; +} + +// Function: detached +inline bool Subflow::detached() const { + return _detached; +} + +// Function: joined +inline bool Subflow::joined() const { + return !_detached; +} + + +// ---------------------------------------------------------------------------- +// Legacy code +// ---------------------------------------------------------------------------- + +using SubflowBuilder = Subflow; + +} // end of namespace tf. --------------------------------------------------- + + diff --git a/bundled/cpp-taskflow/include/taskflow/core/graph.hpp b/bundled/cpp-taskflow/include/taskflow/core/graph.hpp new file mode 100644 index 0000000000..b69cdf84fc --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/graph.hpp @@ -0,0 +1,529 @@ +#pragma once + +#include "error.hpp" +#include "../declarations.hpp" +#include "../utility/object_pool.hpp" +#include "../utility/traits.hpp" +#include "../utility/passive_vector.hpp" +#include "../utility/singleton.hpp" +#include "../utility/uuid.hpp" +#include "../utility/os.hpp" +#include "../nstd/variant.hpp" + +#if defined(__CUDA__) || defined(__CUDACC__) +#define TF_ENABLE_CUDA +#include "../cuda/cuda_flow.hpp" +#endif + +namespace tf { + +// ---------------------------------------------------------------------------- +// domain +// ---------------------------------------------------------------------------- + +enum Domain : int { + HOST = 0, +#ifdef TF_ENABLE_CUDA + CUDA, +#endif + NUM_DOMAINS +}; + + +// ---------------------------------------------------------------------------- +// Class: Graph +// ---------------------------------------------------------------------------- +class Graph { + + friend class Node; + friend class Taskflow; + friend class Executor; + + public: + + Graph() = default; + Graph(const Graph&) = delete; + Graph(Graph&&); + + ~Graph(); + + Graph& operator = (const Graph&) = delete; + Graph& operator = (Graph&&); + + void clear(); + + bool empty() const; + + size_t size() const; + + template + Node* emplace_back(Args&& ...); + + Node* emplace_back(); + + private: + + static ObjectPool& _node_pool(); + + std::vector _nodes; +}; + +// ---------------------------------------------------------------------------- + +// Class: Node +class Node { + + friend class Task; + friend class TaskView; + friend class Topology; + friend class Taskflow; + friend class Executor; + friend class FlowBuilder; + friend class Subflow; + + TF_ENABLE_POOLABLE_ON_THIS; + + // state bit flag + constexpr static int SPAWNED = 0x1; + constexpr static int BRANCH = 0x2; + + // static work handle + struct StaticWork { + + template + StaticWork(C&&); + + std::function work; + }; + + // dynamic work handle + struct DynamicWork { + + template + DynamicWork(C&&); + + std::function work; + Graph subgraph; + }; + + // condition work handle + struct ConditionWork { + + template + ConditionWork(C&&); + + std::function work; + }; + + // module work handle + struct ModuleWork { + + template + ModuleWork(T&&); + + Taskflow* module {nullptr}; + }; + + // cudaFlow work handle +#ifdef TF_ENABLE_CUDA + struct cudaFlowWork { + + template + cudaFlowWork(C&& c) : work {std::forward(c)} {} + + std::function work; + + cudaGraph graph; + }; +#endif + + using handle_t = nstd::variant< + nstd::monostate, // placeholder +#ifdef TF_ENABLE_CUDA + cudaFlowWork, // cudaFlow +#endif + StaticWork, // static tasking + DynamicWork, // dynamic tasking + ConditionWork, // conditional tasking + ModuleWork // composable tasking + >; + + public: + + // variant index + constexpr static auto PLACEHOLDER_WORK = get_index_v; + constexpr static auto STATIC_WORK = get_index_v; + constexpr static auto DYNAMIC_WORK = get_index_v; + constexpr static auto CONDITION_WORK = get_index_v; + constexpr static auto MODULE_WORK = get_index_v; + +#ifdef TF_ENABLE_CUDA + constexpr static auto CUDAFLOW_WORK = get_index_v; +#endif + + template + Node(Args&&... args); + + ~Node(); + + size_t num_successors() const; + size_t num_dependents() const; + size_t num_strong_dependents() const; + size_t num_weak_dependents() const; + + const std::string& name() const; + + Domain domain() const; + + private: + + std::string _name; + + handle_t _handle; + + PassiveVector _successors; + PassiveVector _dependents; + + Topology* _topology {nullptr}; + + Node* _parent {nullptr}; + + int _state {0}; + + std::atomic _join_counter {0}; + + void _precede(Node*); + void _set_state(int); + void _unset_state(int); + void _clear_state(); + void _set_up_join_counter(); + + bool _has_state(int) const; + +}; + +// ---------------------------------------------------------------------------- +// Definition for Node::StaticWork +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::StaticWork::StaticWork(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::DynamicWork +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::DynamicWork::DynamicWork(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::ConditionWork +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::ConditionWork::ConditionWork(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::ModuleWork +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::ModuleWork::ModuleWork(T&& tf) : module {tf} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::Node(Args&&... args): _handle{std::forward(args)...} { +} + +// Destructor +inline Node::~Node() { + // this is to avoid stack overflow + + if(_handle.index() == DYNAMIC_WORK) { + + auto& subgraph = nstd::get(_handle).subgraph; + + std::vector nodes; + + std::move( + subgraph._nodes.begin(), subgraph._nodes.end(), std::back_inserter(nodes) + ); + subgraph._nodes.clear(); + + size_t i = 0; + + while(i < nodes.size()) { + + if(nodes[i]->_handle.index() == DYNAMIC_WORK) { + + auto& sbg = nstd::get(nodes[i]->_handle).subgraph; + std::move( + sbg._nodes.begin(), sbg._nodes.end(), std::back_inserter(nodes) + ); + sbg._nodes.clear(); + } + + ++i; + } + + auto& np = Graph::_node_pool(); + for(i=0; i~Node(); + //np.deallocate(nodes[i]); + np.recycle(nodes[i]); + } + } +} + +// Procedure: _precede +inline void Node::_precede(Node* v) { + _successors.push_back(v); + v->_dependents.push_back(this); +} + +// Function: num_successors +inline size_t Node::num_successors() const { + return _successors.size(); +} + +// Function: dependents +inline size_t Node::num_dependents() const { + return _dependents.size(); +} + +// Function: num_weak_dependents +inline size_t Node::num_weak_dependents() const { + return std::count_if( + _dependents.begin(), + _dependents.end(), + [](Node* node){ return node->_handle.index() == Node::CONDITION_WORK; } + ); +} + +// Function: num_strong_dependents +inline size_t Node::num_strong_dependents() const { + return std::count_if( + _dependents.begin(), + _dependents.end(), + [](Node* node){ return node->_handle.index() != Node::CONDITION_WORK; } + ); +} + +// Function: name +inline const std::string& Node::name() const { + return _name; +} + +// Function: domain +inline Domain Node::domain() const { + + Domain domain; + + switch(_handle.index()) { + + case STATIC_WORK: + case DYNAMIC_WORK: + case CONDITION_WORK: + case MODULE_WORK: + domain = Domain::HOST; + break; + +#ifdef TF_ENABLE_CUDA + case CUDAFLOW_WORK: + domain = Domain::CUDA; + break; +#endif + + default: + domain = Domain::HOST; + break; + } + + return domain; +} + +// +//// Function: dump +//inline std::string Node::dump() const { +// std::ostringstream os; +// dump(os); +// return os.str(); +//} +// +//// Function: dump +//inline void Node::dump(std::ostream& os) const { +// +// os << 'p' << this << "[label=\""; +// if(_name.empty()) os << 'p' << this; +// else os << _name; +// os << "\" "; +// +// // condition node is colored green +// if(_handle.index() == CONDITION_WORK) { +// os << " shape=diamond color=black fillcolor=aquamarine style=filled"; +// } +// +// os << "];\n"; +// +// for(size_t s=0; s<_successors.size(); ++s) { +// if(_handle.index() == CONDITION_WORK) { +// // case edge is dashed +// os << 'p' << this << " -> p" << _successors[s] +// << " [style=dashed label=\"" << s << "\"];\n"; +// } +// else { +// os << 'p' << this << " -> p" << _successors[s] << ";\n"; +// } +// } +// +// // subflow join node +// if(_parent && _successors.size() == 0) { +// os << 'p' << this << " -> p" << _parent << ";\n"; +// } +// +// if(_subgraph && !_subgraph->empty()) { +// +// os << "subgraph cluster_p" << this << " {\nlabel=\"Subflow: "; +// if(_name.empty()) os << 'p' << this; +// else os << _name; +// +// os << "\";\n" << "color=blue\n"; +// +// for(const auto& n : _subgraph->nodes()) { +// n->dump(os); +// } +// os << "}\n"; +// } +//} + +// Procedure: _set_state +inline void Node::_set_state(int flag) { + _state |= flag; +} + +// Procedure: _unset_state +inline void Node::_unset_state(int flag) { + _state &= ~flag; +} + +// Procedure: _clear_state +inline void Node::_clear_state() { + _state = 0; +} + +// Procedure: _set_up_join_counter +inline void Node::_set_up_join_counter() { + + int c = 0; + + for(auto p : _dependents) { + if(p->_handle.index() == Node::CONDITION_WORK) { + _set_state(Node::BRANCH); + } + else { + c++; + } + } + + _join_counter.store(c, std::memory_order_relaxed); +} + +// Function: _has_state +inline bool Node::_has_state(int flag) const { + return _state & flag; +} + +// ---------------------------------------------------------------------------- +// Graph definition +// ---------------------------------------------------------------------------- + +// Function: _node_pool +inline ObjectPool& Graph::_node_pool() { + static ObjectPool pool; + return pool; +} + +// Destructor +inline Graph::~Graph() { + auto& np = _node_pool(); + for(auto node : _nodes) { + //node->~Node(); + //np.deallocate(node); + np.recycle(node); + } +} + +// Move constructor +inline Graph::Graph(Graph&& other) : + _nodes {std::move(other._nodes)} { +} + +// Move assignment +inline Graph& Graph::operator = (Graph&& other) { + _nodes = std::move(other._nodes); + return *this; +} + +// Procedure: clear +inline void Graph::clear() { + auto& np = _node_pool(); + for(auto node : _nodes) { + //node->~Node(); + //np.deallocate(node); + np.recycle(node); + } + _nodes.clear(); +} + +// Function: size +// query the size +inline size_t Graph::size() const { + return _nodes.size(); +} + +// Function: empty +// query the emptiness +inline bool Graph::empty() const { + return _nodes.empty(); +} + +// Function: emplace_back +// create a node from a give argument; constructor is called if necessary +template +Node* Graph::emplace_back(ArgsT&&... args) { + //auto node = _node_pool().allocate(); + //new (node) Node(std::forward(args)...); + //_nodes.push_back(node); + _nodes.push_back(_node_pool().animate(std::forward(args)...)); + return _nodes.back(); +} + +// Function: emplace_back +// create a node from a give argument; constructor is called if necessary +inline Node* Graph::emplace_back() { + //auto node = _node_pool().allocate(); + //new (node) Node(); + //_nodes.push_back(node); + _nodes.push_back(_node_pool().animate()); + return _nodes.back(); +} + + +} // end of namespace tf. --------------------------------------------------- + + + + + diff --git a/bundled/cpp-taskflow/include/taskflow/core/notifier.hpp b/bundled/cpp-taskflow/include/taskflow/core/notifier.hpp new file mode 100644 index 0000000000..b3bdb9ff69 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/notifier.hpp @@ -0,0 +1,274 @@ +// 2019/02/09 - created by Tsung-Wei Huang +// - modified the event count from Eigen + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Dmitry Vyukov +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +namespace tf { + +// Notifier allows to wait for arbitrary predicates in non-blocking +// algorithms. Think of condition variable, but wait predicate does not need to +// be protected by a mutex. Usage: +// Waiting thread does: +// +// if (predicate) +// return act(); +// Notifier::Waiter& w = waiters[my_index]; +// ec.prepare_wait(&w); +// if (predicate) { +// ec.cancel_wait(&w); +// return act(); +// } +// ec.commit_wait(&w); +// +// Notifying thread does: +// +// predicate = true; +// ec.notify(true); +// +// notify is cheap if there are no waiting threads. prepare_wait/commit_wait are not +// cheap, but they are executed only if the preceeding predicate check has +// failed. +// +// Algorihtm outline: +// There are two main variables: predicate (managed by user) and _state. +// Operation closely resembles Dekker mutual algorithm: +// https://en.wikipedia.org/wiki/Dekker%27s_algorithm +// Waiting thread sets _state then checks predicate, Notifying thread sets +// predicate then checks _state. Due to seq_cst fences in between these +// operations it is guaranteed than either waiter will see predicate change +// and won't block, or notifying thread will see _state change and will unblock +// the waiter, or both. But it can't happen that both threads don't see each +// other changes, which would lead to deadlock. +class Notifier { + + friend class Executor; + + public: + + struct Waiter { + std::atomic next; + std::mutex mu; + std::condition_variable cv; + uint64_t epoch; + unsigned state; + enum { + kNotSignaled, + kWaiting, + kSignaled, + }; + }; + + explicit Notifier(size_t N) : _waiters{N} { + assert(_waiters.size() < (1 << kWaiterBits) - 1); + // Initialize epoch to something close to overflow to test overflow. + _state = kStackMask | (kEpochMask - kEpochInc * _waiters.size() * 2); + } + + ~Notifier() { + // Ensure there are no waiters. + assert((_state.load() & (kStackMask | kWaiterMask)) == kStackMask); + } + + // prepare_wait prepares for waiting. + // After calling this function the thread must re-check the wait predicate + // and call either cancel_wait or commit_wait passing the same Waiter object. + void prepare_wait(Waiter* w) { + w->epoch = _state.fetch_add(kWaiterInc, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + // commit_wait commits waiting. + void commit_wait(Waiter* w) { + w->state = Waiter::kNotSignaled; + // Modification epoch of this waiter. + uint64_t epoch = + (w->epoch & kEpochMask) + + (((w->epoch & kWaiterMask) >> kWaiterShift) << kEpochShift); + uint64_t state = _state.load(std::memory_order_seq_cst); + for (;;) { + if (int64_t((state & kEpochMask) - epoch) < 0) { + // The preceeding waiter has not decided on its fate. Wait until it + // calls either cancel_wait or commit_wait, or is notified. + std::this_thread::yield(); + state = _state.load(std::memory_order_seq_cst); + continue; + } + // We've already been notified. + if (int64_t((state & kEpochMask) - epoch) > 0) return; + // Remove this thread from prewait counter and add it to the waiter list. + assert((state & kWaiterMask) != 0); + uint64_t newstate = state - kWaiterInc + kEpochInc; + newstate = (newstate & ~kStackMask) | (w - &_waiters[0]); + if ((state & kStackMask) == kStackMask) + w->next.store(nullptr, std::memory_order_relaxed); + else + w->next.store(&_waiters[state & kStackMask], std::memory_order_relaxed); + if (_state.compare_exchange_weak(state, newstate, + std::memory_order_release)) + break; + } + _park(w); + } + + // cancel_wait cancels effects of the previous prepare_wait call. + void cancel_wait(Waiter* w) { + uint64_t epoch = + (w->epoch & kEpochMask) + + (((w->epoch & kWaiterMask) >> kWaiterShift) << kEpochShift); + uint64_t state = _state.load(std::memory_order_relaxed); + for (;;) { + if (int64_t((state & kEpochMask) - epoch) < 0) { + // The preceeding waiter has not decided on its fate. Wait until it + // calls either cancel_wait or commit_wait, or is notified. + std::this_thread::yield(); + state = _state.load(std::memory_order_relaxed); + continue; + } + // We've already been notified. + if (int64_t((state & kEpochMask) - epoch) > 0) return; + // Remove this thread from prewait counter. + assert((state & kWaiterMask) != 0); + if (_state.compare_exchange_weak(state, state - kWaiterInc + kEpochInc, + std::memory_order_relaxed)) + return; + } + } + + // notify wakes one or all waiting threads. + // Must be called after changing the associated wait predicate. + void notify(bool all) { + std::atomic_thread_fence(std::memory_order_seq_cst); + uint64_t state = _state.load(std::memory_order_acquire); + for (;;) { + // Easy case: no waiters. + if ((state & kStackMask) == kStackMask && (state & kWaiterMask) == 0) + return; + uint64_t waiters = (state & kWaiterMask) >> kWaiterShift; + uint64_t newstate; + if (all) { + // Reset prewait counter and empty wait list. + newstate = (state & kEpochMask) + (kEpochInc * waiters) + kStackMask; + } else if (waiters) { + // There is a thread in pre-wait state, unblock it. + newstate = state + kEpochInc - kWaiterInc; + } else { + // Pop a waiter from list and unpark it. + Waiter* w = &_waiters[state & kStackMask]; + Waiter* wnext = w->next.load(std::memory_order_relaxed); + uint64_t next = kStackMask; + if (wnext != nullptr) next = wnext - &_waiters[0]; + // Note: we don't add kEpochInc here. ABA problem on the lock-free stack + // can't happen because a waiter is re-pushed onto the stack only after + // it was in the pre-wait state which inevitably leads to epoch + // increment. + newstate = (state & kEpochMask) + next; + } + if (_state.compare_exchange_weak(state, newstate, + std::memory_order_acquire)) { + if (!all && waiters) return; // unblocked pre-wait thread + if ((state & kStackMask) == kStackMask) return; + Waiter* w = &_waiters[state & kStackMask]; + if (!all) w->next.store(nullptr, std::memory_order_relaxed); + _unpark(w); + return; + } + } + } + + // notify n workers + void notify_n(size_t n) { + if(n >= _waiters.size()) { + notify(true); + } + else { + for(size_t k=0; k _state; + std::vector _waiters; + + void _park(Waiter* w) { + std::unique_lock lock(w->mu); + while (w->state != Waiter::kSignaled) { + w->state = Waiter::kWaiting; + w->cv.wait(lock); + } + } + + void _unpark(Waiter* waiters) { + Waiter* next = nullptr; + for (Waiter* w = waiters; w; w = next) { + next = w->next.load(std::memory_order_relaxed); + unsigned state; + { + std::unique_lock lock(w->mu); + state = w->state; + w->state = Waiter::kSignaled; + } + // Avoid notifying if it wasn't waiting. + if (state == Waiter::kWaiting) w->cv.notify_one(); + } + } + + Notifier(const Notifier&) = delete; + Notifier& operator=(const Notifier&) = delete; + + Notifier(Notifier&& rhs) : + _state {rhs._state.load()}, + _waiters {std::move(rhs._waiters)} { + } + + +}; + + + +} // namespace tf ------------------------------------------------------------ + diff --git a/bundled/cpp-taskflow/include/taskflow/core/observer.hpp b/bundled/cpp-taskflow/include/taskflow/core/observer.hpp new file mode 100644 index 0000000000..02e5d4f1e5 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/observer.hpp @@ -0,0 +1,504 @@ +// 2020/04/30 - midified by Tsung-Wei Huang +// - adding TaskflowBoard support +// +// 2019/07/31 - modified by Tsung-Wei Huang +// - fixed the missing comma in outputing JSON +// +// 2019/06/13 - modified by Tsung-Wei Huang +// - added TaskView interface +// +// 2019/04/17 - created by Tsung-Wei Huang + +#pragma once + +#include "task.hpp" + +namespace tf { + +/** +@class: ObserverInterface + +@brief The interface class for creating an executor observer. + +The tf::ExecutorObserver class let users define methods to monitor the behaviors +of an executor. +This is particularly useful when you want to inspect the performance of an executor. +*/ +class ObserverInterface { + + friend class Executor; + + public: + + /** + @brief virtual destructor + */ + virtual ~ObserverInterface() = default; + + /** + @brief constructor-like method to call when the executor observer is fully created + @param num_workers the number of the worker threads in the executor + */ + virtual void set_up(size_t num_workers) = 0; + + /** + @brief method to call before a worker thread executes a closure + @param worker_id the id of this worker thread + @param task_view a constant wrapper object to the task + */ + virtual void on_entry(size_t worker_id, TaskView task_view) = 0; + + /** + @brief method to call after a worker thread executed a closure + @param worker_id the id of this worker thread + @param task_view a constant wrapper object to the task + */ + virtual void on_exit(size_t worker_id, TaskView task_view) = 0; +}; + +// ---------------------------------------------------------------------------- +// ChromeTracingObserver definition +// ---------------------------------------------------------------------------- + +/** +@class: ChromeTracingObserver + +@brief observer designed based on chrome tracing format + +*/ +class ChromeTracingObserver : public ObserverInterface { + + friend class Executor; + + // data structure to record each task execution + struct Segment { + + std::string name; + + std::chrono::time_point beg; + std::chrono::time_point end; + + Segment( + const std::string& n, + std::chrono::time_point b + ); + + Segment( + const std::string& n, + std::chrono::time_point b, + std::chrono::time_point e + ); + }; + + // data structure to store the entire execution timeline + struct Timeline { + std::chrono::time_point origin; + std::vector> segments; + }; + + public: + + /** + @brief dump the timelines in JSON format to an ostream + @param ostream the target std::ostream to dump + */ + inline void dump(std::ostream& ostream) const; + + /** + @brief dump the timelines in JSON to a std::string + @return a JSON string + */ + inline std::string dump() const; + + /** + @brief clear the timeline data + */ + inline void clear(); + + /** + @brief get the number of total tasks in the observer + @return number of total tasks + */ + inline size_t num_tasks() const; + + private: + + inline void set_up(size_t num_workers) override final; + inline void on_entry(size_t worker_id, TaskView task_view) override final; + inline void on_exit(size_t worker_id, TaskView task_view) override final; + + Timeline _timeline; +}; + +// constructor +inline ChromeTracingObserver::Segment::Segment( + const std::string& n, + std::chrono::time_point b +) : + name {n}, beg {b} { +} + +// constructor +inline ChromeTracingObserver::Segment::Segment( + const std::string& n, + std::chrono::time_point b, + std::chrono::time_point e +) : + name {n}, beg {b}, end {e} { +} + +// Procedure: set_up +inline void ChromeTracingObserver::set_up(size_t num_workers) { + + _timeline.segments.resize(num_workers); + + for(size_t w=0; w 0); + _timeline.segments[w].back().end = std::chrono::steady_clock::now(); +} + +// Function: clear +inline void ChromeTracingObserver::clear() { + for(size_t w=0; w<_timeline.segments.size(); ++w) { + _timeline.segments[w].clear(); + } +} + +// Procedure: dump +inline void ChromeTracingObserver::dump(std::ostream& os) const { + + size_t first; + + for(first = 0; first<_timeline.segments.size(); ++first) { + if(_timeline.segments[first].size() > 0) { + break; + } + } + + os << '['; + + for(size_t w=first; w<_timeline.segments.size(); w++) { + + if(w != first && _timeline.segments[w].size() > 0) { + os << ','; + } + + for(size_t i=0; i<_timeline.segments[w].size(); i++) { + + os << '{' + << "\"cat\":\"ChromeTracingObserver\","; + + // name field + os << "\"name\":\""; + if(_timeline.segments[w][i].name.empty()) { + os << w << '_' << i; + } + else { + os << _timeline.segments[w][i].name; + } + os << "\","; + + // segment field + os << "\"ph\":\"X\"," + << "\"pid\":1," + << "\"tid\":" << w << ',' + << "\"ts\":" << std::chrono::duration_cast( + _timeline.segments[w][i].beg - _timeline.origin + ).count() << ',' + << "\"dur\":" << std::chrono::duration_cast( + _timeline.segments[w][i].end - _timeline.segments[w][i].beg + ).count(); + + if(i != _timeline.segments[w].size() - 1) { + os << "},"; + } + else { + os << '}'; + } + } + } + os << "]\n"; +} + +// Function: dump +inline std::string ChromeTracingObserver::dump() const { + std::ostringstream oss; + dump(oss); + return oss.str(); +} + +// Function: num_tasks +inline size_t ChromeTracingObserver::num_tasks() const { + return std::accumulate( + _timeline.segments.begin(), _timeline.segments.end(), size_t{0}, + [](size_t sum, const auto& exe){ + return sum + exe.size(); + } + ); +} + +// ---------------------------------------------------------------------------- +// TFProfObserver definition +// ---------------------------------------------------------------------------- + +/** +@class: TFProfObserver + +@brief observer designed based on taskflow board format + +*/ +class TFProfObserver : public ObserverInterface { + + friend class Executor; + + // data structure to record each task execution + struct Segment { + + std::string name; + TaskType type; + + std::chrono::time_point beg; + std::chrono::time_point end; + + Segment( + const std::string& n, + TaskType t, + std::chrono::time_point b + ); + + Segment( + const std::string& n, + TaskType t, + std::chrono::time_point b, + std::chrono::time_point e + ); + }; + + // data structure to store the entire execution timeline + struct Timeline { + std::chrono::time_point origin; + std::vector> segments; + }; + + public: + + /** + @brief dump the timelines in JSON format to an ostream + @param ostream the target std::ostream to dump + */ + inline void dump(std::ostream& ostream) const; + + /** + @brief dump the timelines in JSON to a std::string + @return a JSON string + */ + inline std::string dump() const; + + /** + @brief clear the timeline data + */ + inline void clear(); + + /** + @brief get the number of total tasks in the observer + @return number of total tasks + */ + inline size_t num_tasks() const; + + private: + + inline void set_up(size_t num_workers) override final; + inline void on_entry(size_t worker_id, TaskView task_view) override final; + inline void on_exit(size_t worker_id, TaskView task_view) override final; + + Timeline _timeline; + + UUID _uuid; +}; + +// constructor +inline TFProfObserver::Segment::Segment( + const std::string& n, + TaskType t, + std::chrono::time_point b +) : + name {n}, type {t}, beg {b} { +} + +// constructor +inline TFProfObserver::Segment::Segment( + const std::string& n, + TaskType t, + std::chrono::time_point b, + std::chrono::time_point e +) : + name {n}, type {t}, beg {b}, end {e} { +} + +// Procedure: set_up +inline void TFProfObserver::set_up(size_t num_workers) { + + _timeline.segments.resize(num_workers); + + for(size_t w=0; w 0); + _timeline.segments[w].back().end = std::chrono::steady_clock::now(); +} + +// Function: clear +inline void TFProfObserver::clear() { + for(size_t w=0; w<_timeline.segments.size(); ++w) { + _timeline.segments[w].clear(); + } +} + +// Procedure: dump +inline void TFProfObserver::dump(std::ostream& os) const { + + size_t first; + + for(first = 0; first<_timeline.segments.size(); ++first) { + if(_timeline.segments[first].size() > 0) { + break; + } + } + + // not timeline data to dump + if(first == _timeline.segments.size()) { + os << "{}\n"; + return; + } + + os << "{\"executor\":\"" << _uuid << "\",\"data\":["; + + for(size_t w=first; w<_timeline.segments.size(); w++) { + + if(_timeline.segments[w].empty()) { + continue; + } + + if(w != first) { + os << ','; + } + + os << "{\"worker\":\"worker " << w << "\",\"data\":["; + for(size_t i=0; i<_timeline.segments[w].size(); ++i) { + + const auto& s = _timeline.segments[w][i]; + + if(i) os << ','; + + // span + os << "{\"span\":[" + << std::chrono::duration_cast( + s.beg - _timeline.origin + ).count() << "," + << std::chrono::duration_cast( + s.end - _timeline.origin + ).count() << "],"; + + // name + os << "\"name\":\""; + if(s.name.empty()) { + os << w << '_' << i; + } + else { + os << s.name; + } + os << "\","; + + // category "type": "Condition Task", + os << "\"type\":\"" << task_type_to_string(s.type) << "\""; + + os << "}"; + } + os << "]}"; + } + + os << "]}\n"; +} + +// Function: dump +inline std::string TFProfObserver::dump() const { + std::ostringstream oss; + dump(oss); + return oss.str(); +} + +// Function: num_tasks +inline size_t TFProfObserver::num_tasks() const { + return std::accumulate( + _timeline.segments.begin(), _timeline.segments.end(), size_t{0}, + [](size_t sum, const auto& exe){ + return sum + exe.size(); + } + ); +} + +// ---------------------------------------------------------------------------- +// Identifier for Each Built-in Observer +// ---------------------------------------------------------------------------- + +/** @enum ObserverType + +built-in observer types + +*/ +enum ObserverType { + TFPROF = 1, + CHROME = 2 +}; + +/** +@brief convert an observer type to a human-readable string +*/ +inline const char* observer_type_to_string(ObserverType type) { + const char* val; + switch(type) { + case TFPROF: val = "TFProf"; break; + case CHROME: val = "Chrome"; break; + default: val = "undefined"; break; + } + return val; +} + +// ---------------------------------------------------------------------------- +// Legacy Alias +// ---------------------------------------------------------------------------- +using ExecutorObserverInterface = ObserverInterface; +using ExecutorObserver = ChromeTracingObserver; + + +} // end of namespace tf ----------------------------------------------------- + + diff --git a/bundled/cpp-taskflow/include/taskflow/core/task.hpp b/bundled/cpp-taskflow/include/taskflow/core/task.hpp new file mode 100644 index 0000000000..589050bf29 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/task.hpp @@ -0,0 +1,750 @@ +#pragma once + +#include "graph.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- +// Task Types +// ---------------------------------------------------------------------------- + +/** +@enum TaskType + +@brief enumeration of all task types +*/ +enum TaskType { + PLACEHOLDER_TASK = Node::PLACEHOLDER_WORK, +#ifdef TF_ENABLE_CUDA + CUDAFLOW_TASK = Node::CUDAFLOW_WORK, +#endif + STATIC_TASK = Node::STATIC_WORK, + DYNAMIC_TASK = Node::DYNAMIC_WORK, + CONDITION_TASK = Node::CONDITION_WORK, + MODULE_TASK = Node::MODULE_WORK, + NUM_TASK_TYPES +}; + +/** +@brief convert a task type to a human-readable string +*/ +inline const char* task_type_to_string(TaskType type) { + + const char* val; + + switch(type) { + case PLACEHOLDER_TASK: val = "placeholder"; break; +#ifdef TF_ENABLE_CUDA + case CUDAFLOW_TASK: val = "cudaflow"; break; +#endif + case STATIC_TASK: val = "static"; break; + case DYNAMIC_TASK: val = "subflow"; break; + case CONDITION_TASK: val = "condition"; break; + case MODULE_TASK: val = "module"; break; + default: val = "undefined"; break; + } + + return val; +} + +// ---------------------------------------------------------------------------- +// Task Traits +// ---------------------------------------------------------------------------- + +/** +@struct is_static_task + +@brief determines if a callable is a static task + +A static task is a callable object constructible from std::function. +*/ +template +constexpr bool is_static_task_v = is_invocable_r_v && + !is_invocable_r_v; + +/** +@struct is_dynamic_task + +@brief determines if a callable is a dynamic task + +A dynamic task is a callable object constructible from std::function. +*/ +template +constexpr bool is_dynamic_task_v = is_invocable_r_v; + +/** +@struct is_condition_task + +@brief determines if a callable is a condition task + +A condition task is a callable object constructible from std::function. +*/ +template +constexpr bool is_condition_task_v = is_invocable_r_v; + +#ifdef TF_ENABLE_CUDA +/** +@struct is_cudaflow_task + +@brief determines if a callable is a cudaflow task + +A cudaFlow task is a callable object constructible from std::function. +*/ +template +constexpr bool is_cudaflow_task_v = is_invocable_r_v; +#endif + + + +// ---------------------------------------------------------------------------- +// Task +// ---------------------------------------------------------------------------- + +/** +@class Task + +@brief handle to a node in a task dependency graph + +A Task is handle object of a node in a dependency graph. +It provides a set of methods for users to access and modify the attributes of +the associated graph node. + +*/ +class Task { + + friend class FlowBuilder; + friend class Taskflow; + friend class TaskView; + + public: + + /** + @brief constructs an empty task + */ + Task() = default; + + /** + @brief constructs the task with the copy of the other task + */ + Task(const Task& other); + + /** + @brief replaces the contents with a copy of the other task + */ + Task& operator = (const Task&); + + /** + @brief replaces the contents with a null pointer + */ + Task& operator = (std::nullptr_t); + + /** + @brief compares if two tasks are associated with the same graph node + */ + bool operator == (const Task& rhs) const; + + /** + @brief compares if two tasks are not associated with the same graph node + */ + bool operator != (const Task& rhs) const; + + /** + @brief queries the name of the task + */ + const std::string& name() const; + + /** + @brief queries the number of successors of the task + */ + size_t num_successors() const; + + /** + @brief queries the number of predecessors of the task + */ + size_t num_dependents() const; + + /** + @brief queries the number of strong dependents of the task + */ + size_t num_strong_dependents() const; + + /** + @brief queries the number of weak dependents of the task + */ + size_t num_weak_dependents() const; + + /** + @brief assigns a name to the task + + @param name a @std_string acceptable string + + @return @c *this + */ + Task& name(const std::string& name); + + /** + @brief assigns a static task + + @tparam C callable object type + + @param callable a callable object constructible from std::function + + @return @c *this + */ + template + std::enable_if_t, Task>& work(C&& callable); + + /** + @brief assigns a dynamic task + + @tparam C callable object type + + @param callable a callable object constructible from std::function + + @return @c *this + */ + template + std::enable_if_t, Task>& work(C&& callable); + + /** + @brief assigns a condition task + + @tparam C callable object type + + @param callable a callable object constructible from std::function + + @return @c *this + */ + template + std::enable_if_t, Task>& work(C&& callable); + +#ifdef TF_ENABLE_CUDA + /** + @brief assigns a cudaFlow task + + @tparam C callable object type + + @param callable a callable object constructible from std::function + + @return @c *this + */ + template + std::enable_if_t, Task>& work(C&& callable); +#endif + + /** + @brief creates a module task from a taskflow + + @param taskflow a taskflow object for the module + + @return @c *this + */ + Task& composed_of(Taskflow& taskflow); + + /** + @brief adds precedence links from this to other tasks + + @tparam Ts... parameter pack + + @param tasks one or multiple tasks + + @return @c *this + */ + template + Task& precede(Ts&&... tasks); + + /** + @brief adds precedence links from other tasks to this + + @tparam Ts parameter pack + + @param tasks one or multiple tasks + + @return @c *this + */ + template + Task& succeed(Ts&&... tasks); + + /** + @brief resets the task handle to null + */ + void reset(); + + /** + @brief resets the associated work to a placeholder + */ + void reset_work(); + + /** + @brief queries if the task handle points to a task node + */ + bool empty() const; + + /** + @brief queries if the task has a work assigned + */ + bool has_work() const; + + /** + @brief applies an visitor callable to each successor of the task + */ + template + void for_each_successor(V&& visitor) const; + + /** + @brief applies an visitor callable to each dependents of the task + */ + template + void for_each_dependent(V&& visitor) const; + + /** + @brief obtains a hash value of the underlying node + */ + size_t hash_value() const; + + /** + @brief returns the task type + */ + TaskType type() const; + + private: + + Task(Node*); + + Node* _node {nullptr}; + + template + void _precede(T&&); + + template + void _precede(T&&, Rest&&...); + + template + void _succeed(T&&); + + template + void _succeed(T&&, Rest&&...); +}; + +// Constructor +inline Task::Task(Node* node) : _node {node} { +} + +// Constructor +inline Task::Task(const Task& rhs) : _node {rhs._node} { +} + +// Function: precede +template +Task& Task::precede(Ts&&... tasks) { + //(_node->_precede(tgts._node), ...); + _precede(std::forward(tasks)...); + return *this; +} + +/// @private +// Procedure: _precede +template +void Task::_precede(T&& other) { + _node->_precede(other._node); +} + +/// @private +// Procedure: _precede +template +void Task::_precede(T&& task, Ts&&... others) { + _precede(std::forward(task)); + _precede(std::forward(others)...); +} + +// Function: succeed +template +Task& Task::succeed(Ts&&... tasks) { + //(tasks._node->_precede(_node), ...); + _succeed(std::forward(tasks)...); + return *this; +} + +/// @private +// Procedure: succeed +template +void Task::_succeed(T&& other) { + other._node->_precede(_node); +} + +/// @private +// Procedure: _succeed +template +void Task::_succeed(T&& task, Ts&&... others) { + _succeed(std::forward(task)); + _succeed(std::forward(others)...); +} + +// Function: composed_of +inline Task& Task::composed_of(Taskflow& tf) { + _node->_handle.emplace(&tf); + return *this; +} + +// Operator = +inline Task& Task::operator = (const Task& rhs) { + _node = rhs._node; + return *this; +} + +// Operator = +inline Task& Task::operator = (std::nullptr_t ptr) { + _node = ptr; + return *this; +} + +// Operator == +inline bool Task::operator == (const Task& rhs) const { + return _node == rhs._node; +} + +// Operator != +inline bool Task::operator != (const Task& rhs) const { + return _node != rhs._node; +} + +// Function: name +inline Task& Task::name(const std::string& name) { + _node->_name = name; + return *this; +} + +// Procedure: reset +inline void Task::reset() { + _node = nullptr; +} + +// Procedure: reset_work +inline void Task::reset_work() { + _node->_handle = nstd::monostate{}; +} + +// Function: name +inline const std::string& Task::name() const { + return _node->_name; +} + +// Function: num_dependents +inline size_t Task::num_dependents() const { + return _node->num_dependents(); +} + +// Function: num_strong_dependents +inline size_t Task::num_strong_dependents() const { + return _node->num_strong_dependents(); +} + +// Function: num_weak_dependents +inline size_t Task::num_weak_dependents() const { + return _node->num_weak_dependents(); +} + +// Function: num_successors +inline size_t Task::num_successors() const { + return _node->num_successors(); +} + +// Function: empty +inline bool Task::empty() const { + return _node == nullptr; +} + +// Function: has_work +inline bool Task::has_work() const { + return _node ? _node->_handle.index() != 0 : false; +} + +// Function: task_type +inline TaskType Task::type() const { + return static_cast(_node->_handle.index()); +} + +// Function: for_each_successor +template +void Task::for_each_successor(V&& visitor) const { + for(size_t i=0; i<_node->_successors.size(); ++i) { + visitor(Task(_node->_successors[i])); + } +} + +// Function: for_each_dependent +template +void Task::for_each_dependent(V&& visitor) const { + for(size_t i=0; i<_node->_dependents.size(); ++i) { + visitor(Task(_node->_dependents[i])); + } +} + +// Function: hash_value +inline size_t Task::hash_value() const { + return std::hash{}(_node); +} + +// Function: work +// assign a static work +template +std::enable_if_t, Task>& Task::work(C&& c) { + _node->_handle.emplace(std::forward(c)); + return *this; +} + +// Function: work +// assigns a dynamic work +template +std::enable_if_t, Task>& Task::work(C&& c) { + _node->_handle.emplace(std::forward(c)); + return *this; +} + +// Function: work +// assigns a condition work +template +std::enable_if_t, Task>& Task::work(C&& c) { + _node->_handle.emplace(std::forward(c)); + return *this; +} + +#ifdef TF_ENABLE_CUDA +// Function: work +// assigns a cudaFlow work +template +std::enable_if_t, Task>& Task::work(C&& c) { + _node->_handle.emplace(std::forward(c)); + return *this; +} +#endif + +// ---------------------------------------------------------------------------- + +/** +@class TaskView + +@brief class to access task information from the observer interface +*/ +class TaskView { + + friend class Executor; + + public: + + /** + @brief constructs an empty task view + */ + TaskView() = default; + + /** + @brief constructs a task view from a task + */ + TaskView(const Task& task); + + /** + @brief constructs the task with the copy of the other task + */ + TaskView(const TaskView& other); + + /** + @brief replaces the contents with a copy of the other task + */ + TaskView& operator = (const TaskView& other); + + /** + @brief replaces the contents with another task + */ + TaskView& operator = (const Task& other); + + /** + @brief replaces the contents with a null pointer + */ + TaskView& operator = (std::nullptr_t); + + /** + @brief compares if two taskviews are associated with the same task + */ + bool operator == (const TaskView&) const; + + /** + @brief compares if two taskviews are associated with different tasks + */ + bool operator != (const TaskView&) const; + + /** + @brief queries the name of the task + */ + const std::string& name() const; + + /** + @brief queries the number of successors of the task + */ + size_t num_successors() const; + + /** + @brief queries the number of predecessors of the task + */ + size_t num_dependents() const; + + /** + @brief queries the number of strong dependents of the task + */ + size_t num_strong_dependents() const; + + /** + @brief queries the number of weak dependents of the task + */ + size_t num_weak_dependents() const; + + /** + @brief resets to an empty view + */ + void reset(); + + /** + @brief queries if the task view is empty + */ + bool empty() const; + + /** + @brief applies an visitor callable to each successor of the task + */ + template + void for_each_successor(V&& visitor) const; + + /** + @brief applies an visitor callable to each dependents of the task + */ + template + void for_each_dependent(V&& visitor) const; + + /** + @brief queries the task type + */ + TaskType type() const; + + private: + + TaskView(Node*); + + Node* _node {nullptr}; +}; + +// Constructor +inline TaskView::TaskView(Node* node) : _node {node} { +} + +// Constructor +inline TaskView::TaskView(const TaskView& rhs) : _node {rhs._node} { +} + +// Constructor +inline TaskView::TaskView(const Task& task) : _node {task._node} { +} + +// Operator = +inline TaskView& TaskView::operator = (const TaskView& rhs) { + _node = rhs._node; + return *this; +} + +// Operator = +inline TaskView& TaskView::operator = (const Task& rhs) { + _node = rhs._node; + return *this; +} + +// Operator = +inline TaskView& TaskView::operator = (std::nullptr_t ptr) { + _node = ptr; + return *this; +} + +// Function: name +inline const std::string& TaskView::name() const { + return _node->_name; +} + +// Function: num_dependents +inline size_t TaskView::num_dependents() const { + return _node->num_dependents(); +} + +// Function: num_strong_dependents +inline size_t TaskView::num_strong_dependents() const { + return _node->num_strong_dependents(); +} + +// Function: num_weak_dependents +inline size_t TaskView::num_weak_dependents() const { + return _node->num_weak_dependents(); +} + +// Function: num_successors +inline size_t TaskView::num_successors() const { + return _node->num_successors(); +} + +// Function: reset +inline void TaskView::reset() { + _node = nullptr; +} + +// Function: empty +inline bool TaskView::empty() const { + return _node == nullptr; +} + +// Function: type +inline TaskType TaskView::type() const { + return static_cast(_node->_handle.index()); +} + +// Operator == +inline bool TaskView::operator == (const TaskView& rhs) const { + return _node == rhs._node; +} + +// Operator != +inline bool TaskView::operator != (const TaskView& rhs) const { + return _node != rhs._node; +} + +// Function: for_each_successor +template +void TaskView::for_each_successor(V&& visitor) const { + for(size_t i=0; i<_node->_successors.size(); ++i) { + visitor(TaskView(_node->_successors[i])); + } +} + +// Function: for_each_dependent +template +void TaskView::for_each_dependent(V&& visitor) const { + for(size_t i=0; i<_node->_dependents.size(); ++i) { + visitor(TaskView(_node->_dependents[i])); + } +} + +} // end of namespace tf. --------------------------------------------------- + +namespace std { + +/** +@class hash + +@brief hash specialization for std::hash + +*/ +template <> +struct hash { + auto operator() (const tf::Task& task) const noexcept { + return task.hash_value(); + } +}; + +} // end of namespace std ---------------------------------------------------- + + + diff --git a/bundled/cpp-taskflow/include/taskflow/core/taskflow.hpp b/bundled/cpp-taskflow/include/taskflow/core/taskflow.hpp new file mode 100644 index 0000000000..f6123eff5d --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/taskflow.hpp @@ -0,0 +1,353 @@ +#pragma once + +#include + +#include "flow_builder.hpp" +#include "topology.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- + +/** +@class Taskflow + +@brief main entry to create a task dependency graph + +*/ +class Taskflow : public FlowBuilder { + + friend class Topology; + friend class Executor; + friend class FlowBuilder; + + struct Dumper { + std::stack stack; + std::unordered_set visited; + }; + + public: + + /** + @brief constructs a taskflow with a given name + */ + Taskflow(const std::string& name); + + /** + @brief constructs a taskflow + */ + Taskflow(); + + /** + @brief destroy the taskflow (virtual call) + */ + virtual ~Taskflow(); + + /** + @brief dumps the taskflow to a std::ostream in DOT format + + @param ostream a std::ostream target + */ + void dump(std::ostream& ostream) const; + + /** + @brief dumps the taskflow in DOT format to a std::string + */ + std::string dump() const; + + /** + @brief queries the number of tasks in the taskflow + */ + size_t num_tasks() const; + + /** + @brief queries the emptiness of the taskflow + */ + bool empty() const; + + /** + @brief sets the name of the taskflow + + @return @c *this + */ + void name(const std::string&); + + /** + @brief queries the name of the taskflow + */ + const std::string& name() const ; + + /** + @brief clears the associated task dependency graph + */ + void clear(); + + /** + @brief applies an visitor callable to each task in the taskflow + */ + template + void for_each_task(V&& visitor) const; + + private: + + std::string _name; + + Graph _graph; + + std::mutex _mtx; + + std::list _topologies; + + void _dump(std::ostream&, const Taskflow*) const; + void _dump(std::ostream&, const Node*, Dumper&) const; + void _dump(std::ostream&, const Graph&, Dumper&) const; +}; + +// Constructor +inline Taskflow::Taskflow(const std::string& name) : + FlowBuilder {_graph}, + _name {name} { +} + +// Constructor +inline Taskflow::Taskflow() : FlowBuilder{_graph} { +} + +// Destructor +inline Taskflow::~Taskflow() { + assert(_topologies.empty()); +} + +// Procedure: +inline void Taskflow::clear() { + _graph.clear(); +} + +// Function: num_noces +inline size_t Taskflow::num_tasks() const { + return _graph.size(); +} + +// Function: empty +inline bool Taskflow::empty() const { + return _graph.empty(); +} + +// Function: name +inline void Taskflow::name(const std::string &name) { + _name = name; +} + +// Function: name +inline const std::string& Taskflow::name() const { + return _name; +} + +// Function: for_each_task +template +void Taskflow::for_each_task(V&& visitor) const { + for(size_t i=0; i<_graph._nodes.size(); ++i) { + visitor(Task(_graph._nodes[i])); + } +} + +// Procedure: dump +inline std::string Taskflow::dump() const { + std::ostringstream oss; + dump(oss); + return oss.str(); +} + +// Function: dump +inline void Taskflow::dump(std::ostream& os) const { + os << "digraph Taskflow {\n"; + _dump(os, this); + os << "}\n"; +} + +// Procedure: _dump +inline void Taskflow::_dump(std::ostream& os, const Taskflow* top) const { + + Dumper dumper; + + dumper.stack.push(top); + dumper.visited.insert(top); + + while(!dumper.stack.empty()) { + + auto f = dumper.stack.top(); + dumper.stack.pop(); + + os << "subgraph cluster_p" << f << " {\nlabel=\"Taskflow: "; + if(f->_name.empty()) os << 'p' << f; + else os << f->_name; + os << "\";\n"; + _dump(os, f->_graph, dumper); + os << "}\n"; + } +} + +// Procedure: _dump +inline void Taskflow::_dump( + std::ostream& os, const Node* node, Dumper& dumper +) const { + + os << 'p' << node << "[label=\""; + if(node->_name.empty()) os << 'p' << node; + else os << node->_name; + os << "\" "; + + // shape for node + switch(node->_handle.index()) { + + case Node::CONDITION_WORK: + os << "shape=diamond color=black fillcolor=aquamarine style=filled"; + break; + +#ifdef TF_ENABLE_CUDA + case Node::CUDAFLOW_WORK: + os << "shape=folder fillcolor=cyan style=filled"; + break; +#endif + + default: + break; + } + + os << "];\n"; + + for(size_t s=0; s_successors.size(); ++s) { + if(node->_handle.index() == Node::CONDITION_WORK) { + // case edge is dashed + os << 'p' << node << " -> p" << node->_successors[s] + << " [style=dashed label=\"" << s << "\"];\n"; + } + else { + os << 'p' << node << " -> p" << node->_successors[s] << ";\n"; + } + } + + // subflow join node + if(node->_parent && node->_successors.size() == 0) { + os << 'p' << node << " -> p" << node->_parent << ";\n"; + } + + switch(node->_handle.index()) { + + case Node::DYNAMIC_WORK: { + auto& sbg = nstd::get(node->_handle).subgraph; + if(!sbg.empty()) { + os << "subgraph cluster_p" << node << " {\nlabel=\"Subflow: "; + if(node->_name.empty()) os << 'p' << node; + else os << node->_name; + + os << "\";\n" << "color=blue\n"; + _dump(os, sbg, dumper); + os << "}\n"; + } + } + break; + +#ifdef TF_ENABLE_CUDA + case Node::CUDAFLOW_WORK: { + auto& cfg = nstd::get(node->_handle).graph; + if(!cfg.empty()) { + os << "subgraph cluster_p" << node << " {\nlabel=\"cudaFlow: "; + if(node->_name.empty()) os << 'p' << node; + else os << node->_name; + + os << "\";\n" << "color=\"purple\"\n"; + + for(const auto& v : cfg._nodes) { + + os << 'p' << v.get() << "[label=\""; + if(v->_name.empty()) { + os << 'p' << v.get() << "\""; + } + else { + os << v->_name << "\""; + } + + switch(v->_handle.index()) { + case cudaNode::NOOP: + break; + + case cudaNode::COPY: + //os << " shape=\"cds\""; + break; + + case cudaNode::KERNEL: + os << " style=\"filled\"" + << " color=\"white\" fillcolor=\"black\"" + << " fontcolor=\"white\"" + << " shape=\"box3d\""; + break; + + default: + break; + } + + os << "];\n"; + for(const auto s : v->_successors) { + os << 'p' << v.get() << " -> " << 'p' << s << ";\n"; + } + + if(v->_successors.size() == 0) { + os << 'p' << v.get() << " -> p" << node << ";\n"; + } + + } + os << "}\n"; + } + } + break; +#endif + + default: + break; + } +} + +// Procedure: _dump +inline void Taskflow::_dump( + std::ostream& os, const Graph& graph, Dumper& dumper +) const { + + for(const auto& n : graph._nodes) { + + // regular task + if(n->_handle.index() != Node::MODULE_WORK) { + _dump(os, n, dumper); + } + // module task + else { + + auto module = nstd::get(n->_handle).module; + + os << 'p' << n << "[shape=box3d, color=blue, label=\""; + if(n->_name.empty()) os << n; + else os << n->_name; + os << " [Taskflow: "; + if(module->_name.empty()) os << 'p' << module; + else os << module->_name; + os << "]\"];\n"; + + if(dumper.visited.find(module) == dumper.visited.end()) { + dumper.visited.insert(module); + dumper.stack.push(module); + } + + for(const auto s : n->_successors) { + os << 'p' << n << "->" << 'p' << s << ";\n"; + } + } + } +} + +// ---------------------------------------------------------------------------- +// Backward compatibility +// ---------------------------------------------------------------------------- +using Framework = Taskflow; + +} // end of namespace tf. --------------------------------------------------- + diff --git a/bundled/cpp-taskflow/include/taskflow/core/topology.hpp b/bundled/cpp-taskflow/include/taskflow/core/topology.hpp new file mode 100644 index 0000000000..97780d9cda --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/topology.hpp @@ -0,0 +1,93 @@ +#pragma once + +//#include "taskflow.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- + +// class: Topology +class Topology { + + friend class Taskflow; + friend class Executor; + + public: + + template + Topology(Taskflow&, P&&, C&&); + + private: + + Taskflow& _taskflow; + + std::promise _promise; + + PassiveVector _sources; + + std::function _pred; + std::function _call; + + std::atomic _join_counter {0}; +}; + +// Constructor +template +inline Topology::Topology(Taskflow& tf, P&& p, C&& c): + _taskflow(tf), + _pred {std::forward

(p)}, + _call {std::forward(c)} { +} + +// Procedure: _bind +// Re-builds the source links and the sink number for this topology. +//inline void Topology::_bind(Graph& g) { +// +// _sources.clear(); +// +// //PassiveVector condition_nodes; +// +// // scan each node in the graph and build up the links +// for(auto& node : g.nodes()) { +// +// node->_topology = this; +// node->_clear_state(); +// node->_set_up_join_counter(); +// +// if(node->num_dependents() == 0) { +// _sources.push_back(node.get()); +// } +// +// //int join_counter = 0; +// //for(auto p : node->_dependents) { +// // if(p->_work.index() == Node::CONDITION_WORK) { +// // node->_set_state(Node::BRANCH); +// // } +// // else { +// // join_counter++; +// // } +// //} +// +// //node->_join_counter.store(join_counter, std::memory_order_relaxed); +// +// //// TODO: Merge with the loop below? +// //if(node->_work.index() == Node::CONDITION_WORK) { +// // condition_nodes.push_back(node.get()); +// //} +// +// //// Reset each node's num_dependents +// //node->_join_counter.store(node->_dependents.size(), std::memory_order_relaxed); +// } +// +// // We need to deduct the condition predecessors in impure case nodes +// //for(auto& n: condition_nodes) { +// // for(auto& s: n->_successors) { +// // s->_join_counter.fetch_sub(1, std::memory_order_relaxed); +// // s->set_branch(); +// // } +// //} +//} + + + +} // end of namespace tf. ---------------------------------------------------- diff --git a/bundled/cpp-taskflow/include/taskflow/core/tsq.hpp b/bundled/cpp-taskflow/include/taskflow/core/tsq.hpp new file mode 100644 index 0000000000..23ea34d21b --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/core/tsq.hpp @@ -0,0 +1,250 @@ +// 2020/02/24 - created by twhuang +// - specialized work stealing queue for pointer + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace tf { + +/** +@class: TaskQueue + +@tparam T data type (must be a pointer) + +@brief Lock-free unbounded single-producer multiple-consumer queue. + +This class implements the work stealing queue described in the paper, +"Correct and Efficient Work-Stealing for Weak Memory Models," +available at https://www.di.ens.fr/~zappa/readings/ppopp13.pdf. + +Only the queue owner can perform pop and push operations, +while others can steal data from the queue. +*/ +template +class TaskQueue { + + static_assert(std::is_pointer::value, "T must be a pointer type"); + + struct Array { + + int64_t C; + int64_t M; + std::atomic* S; + + explicit Array(int64_t c) : + C {c}, + M {c-1}, + S {new std::atomic[static_cast(C)]} { + } + + ~Array() { + delete [] S; + } + + int64_t capacity() const noexcept { + return C; + } + + template + void push(int64_t i, O&& o) noexcept { + S[i & M].store(std::forward(o), std::memory_order_relaxed); + } + + T pop(int64_t i) noexcept { + return S[i & M].load(std::memory_order_relaxed); + } + + Array* resize(int64_t b, int64_t t) { + Array* ptr = new Array {2*C}; + for(int64_t i=t; i!=b; ++i) { + ptr->push(i, pop(i)); + } + return ptr; + } + + }; + + std::atomic _top; + std::atomic _bottom; + std::atomic _array; + std::vector _garbage; + + public: + + /** + @brief constructs the queue with a given capacity + + @param capacity the capacity of the queue (must be power of 2) + */ + explicit TaskQueue(int64_t capacity = 1024); + + /** + @brief destructs the queue + */ + ~TaskQueue(); + + /** + @brief queries if the queue is empty at the time of this call + */ + bool empty() const noexcept; + + /** + @brief queries the number of items at the time of this call + */ + size_t size() const noexcept; + + /** + @brief queries the capacity of the queue + */ + int64_t capacity() const noexcept; + + /** + @brief inserts an item to the queue + + Only the owner thread can insert an item to the queue. + The operation can trigger the queue to resize its capacity + if more space is required. + + @tparam O data type + + @param item the item to perfect-forward to the queue + */ + void push(T item); + + /** + @brief pops out an item from the queue + + Only the owner thread can pop out an item from the queue. + The return can be a nullptr if this operation failed (empty queue). + */ + T pop(); + + /** + @brief steals an item from the queue + + Any threads can try to steal an item from the queue. + The return can be a nullptr if this operation failed (not necessary empty). + */ + T steal(); +}; + +// Constructor +template +TaskQueue::TaskQueue(int64_t c) { + assert(c && (!(c & (c-1)))); + _top.store(0, std::memory_order_relaxed); + _bottom.store(0, std::memory_order_relaxed); + _array.store(new Array{c}, std::memory_order_relaxed); + _garbage.reserve(32); +} + +// Destructor +template +TaskQueue::~TaskQueue() { + for(auto a : _garbage) { + delete a; + } + delete _array.load(); +} + +// Function: empty +template +bool TaskQueue::empty() const noexcept { + int64_t b = _bottom.load(std::memory_order_relaxed); + int64_t t = _top.load(std::memory_order_relaxed); + return b <= t; +} + +// Function: size +template +size_t TaskQueue::size() const noexcept { + int64_t b = _bottom.load(std::memory_order_relaxed); + int64_t t = _top.load(std::memory_order_relaxed); + return static_cast(b >= t ? b - t : 0); +} + +// Function: push +template +void TaskQueue::push(T o) { + int64_t b = _bottom.load(std::memory_order_relaxed); + int64_t t = _top.load(std::memory_order_acquire); + Array* a = _array.load(std::memory_order_relaxed); + + // queue is full + if(a->capacity() - 1 < (b - t)) { + Array* tmp = a->resize(b, t); + _garbage.push_back(a); + std::swap(a, tmp); + _array.store(a, std::memory_order_relaxed); + } + + a->push(b, o); + std::atomic_thread_fence(std::memory_order_release); + _bottom.store(b + 1, std::memory_order_relaxed); +} + +// Function: pop +template +T TaskQueue::pop() { + int64_t b = _bottom.load(std::memory_order_relaxed) - 1; + Array* a = _array.load(std::memory_order_relaxed); + _bottom.store(b, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_seq_cst); + int64_t t = _top.load(std::memory_order_relaxed); + + T item {nullptr}; + + if(t <= b) { + item = a->pop(b); + if(t == b) { + // the last item just got stolen + if(!_top.compare_exchange_strong(t, t+1, + std::memory_order_seq_cst, + std::memory_order_relaxed)) { + item = nullptr; + } + _bottom.store(b + 1, std::memory_order_relaxed); + } + } + else { + _bottom.store(b + 1, std::memory_order_relaxed); + } + + return item; +} + +// Function: steal +template +T TaskQueue::steal() { + int64_t t = _top.load(std::memory_order_acquire); + std::atomic_thread_fence(std::memory_order_seq_cst); + int64_t b = _bottom.load(std::memory_order_acquire); + + T item {nullptr}; + + if(t < b) { + Array* a = _array.load(std::memory_order_consume); + item = a->pop(t); + if(!_top.compare_exchange_strong(t, t+1, + std::memory_order_seq_cst, + std::memory_order_relaxed)) { + return nullptr; + } + } + + return item; +} + +// Function: capacity +template +int64_t TaskQueue::capacity() const noexcept { + return _array.load(std::memory_order_relaxed)->capacity(); +} + +} // end of namespace tf ----------------------------------------------------- diff --git a/bundled/cpp-taskflow/include/taskflow/cuda/cuda_device.hpp b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_device.hpp new file mode 100644 index 0000000000..7b9371a97c --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_device.hpp @@ -0,0 +1,136 @@ +#pragma once + +#include "cuda_error.hpp" + +namespace tf { + +/** +@brief queries the number of available devices +*/ +inline size_t cuda_num_devices() { + int N = 0; + TF_CHECK_CUDA(cudaGetDeviceCount(&N), "failed to get device count"); + return static_cast(N); +} + +/** +@brief gets the current device associated with the caller thread +*/ +inline int cuda_get_device() { + int id; + TF_CHECK_CUDA(cudaGetDevice(&id), "failed to get current device id"); + return id; +} + +/** +@brief switches to a given device context +*/ +inline void cuda_set_device(int id) { + TF_CHECK_CUDA(cudaSetDevice(id), "failed to switch to device ", id); +} + +/** +@brief obtains the device property +*/ +inline void cuda_get_device_property(int i, cudaDeviceProp& p) { + TF_CHECK_CUDA( + cudaGetDeviceProperties(&p, i), "failed to get property of device ", i + ); +} + +/** +@brief obtains the device property +*/ +inline cudaDeviceProp cuda_get_device_property(int i) { + cudaDeviceProp p; + TF_CHECK_CUDA( + cudaGetDeviceProperties(&p, i), "failed to get property of device ", i + ); + return p; +} + +/** +@brief cuda_dump_device_property +*/ +inline void cuda_dump_device_property(std::ostream& os, const cudaDeviceProp& p) { + + os << "Major revision number: " << p.major << '\n' + << "Minor revision number: " << p.minor << '\n' + << "Name: " << p.name << '\n' + << "Total global memory: " << p.totalGlobalMem << '\n' + << "Total shared memory per block: " << p.sharedMemPerBlock << '\n' + << "Total registers per block: " << p.regsPerBlock << '\n' + << "Warp size: " << p.warpSize << '\n' + << "Maximum memory pitch: " << p.memPitch << '\n' + << "Maximum threads per block: " << p.maxThreadsPerBlock << '\n'; + + os << "Maximum dimension of block: "; + for (int i = 0; i < 3; ++i) { + if(i) os << 'x'; + os << p.maxThreadsDim[i]; + } + os << '\n'; + + os << "Maximum dimenstion of grid: "; + for (int i = 0; i < 3; ++i) { + if(i) os << 'x'; + os << p.maxGridSize[i];; + } + os << '\n'; + + os << "Clock rate: " << p.clockRate << '\n' + << "Total constant memory: " << p.totalConstMem << '\n' + << "Texture alignment: " << p.textureAlignment << '\n' + << "Concurrent copy and execution: " << p.deviceOverlap << '\n' + << "Number of multiprocessors: " << p.multiProcessorCount << '\n' + << "Kernel execution timeout: " << p.kernelExecTimeoutEnabled << '\n' + << "GPU sharing Host Memory: " << p.integrated << '\n' + << "Host page-locked mem mapping: " << p.canMapHostMemory << '\n' + << "Alignment for Surfaces: " << p.surfaceAlignment << '\n' + << "Device has ECC support: " << p.ECCEnabled << '\n' + << "Unified Addressing (UVA): " << p.unifiedAddressing << '\n'; +} + +// ---------------------------------------------------------------------------- +// Class definitions +// ---------------------------------------------------------------------------- + +/** @class cudaScopedDevice + +@brief RAII-style device context switch + +*/ +class cudaScopedDevice { + + public: + + cudaScopedDevice(int); + ~cudaScopedDevice(); + + private: + + int _p; +}; + +// Constructor +inline cudaScopedDevice::cudaScopedDevice(int dev) { + TF_CHECK_CUDA(cudaGetDevice(&_p), "failed to get current device scope"); + if(_p == dev) { + _p = -1; + } + else { + TF_CHECK_CUDA(cudaSetDevice(dev), "failed to scope on device ", dev); + } +} + +// Destructor +inline cudaScopedDevice::~cudaScopedDevice() { + if(_p != -1) { + cudaSetDevice(_p); + //TF_CHECK_CUDA(cudaSetDevice(_p), "failed to scope back to device ", _p); + } +} + +} // end of namespace cuda --------------------------------------------------- + + diff --git a/bundled/cpp-taskflow/include/taskflow/cuda/cuda_error.hpp b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_error.hpp new file mode 100644 index 0000000000..a2b679571c --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_error.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +#include "../utility/stringify.hpp" + +#define TF_CUDA_REMOVE_FIRST_HELPER(N, ...) __VA_ARGS__ +#define TF_CUDA_REMOVE_FIRST(...) TF_CUDA_REMOVE_FIRST_HELPER(__VA_ARGS__) +#define TF_CUDA_GET_FIRST_HELPER(N, ...) N +#define TF_CUDA_GET_FIRST(...) TF_CUDA_GET_FIRST_HELPER(__VA_ARGS__) + +#define TF_CHECK_CUDA(...) \ +if(TF_CUDA_GET_FIRST(__VA_ARGS__) != cudaSuccess) { \ + std::ostringstream oss; \ + auto ev = TF_CUDA_GET_FIRST(__VA_ARGS__); \ + auto unknown_str = "unknown error"; \ + auto unknown_name = "cudaErrorUnknown"; \ + auto error_str = ::cudaGetErrorString(ev); \ + auto error_name = ::cudaGetErrorName(ev); \ + oss << "[" << __FILE__ << ":" << __LINE__ << "] " \ + << (error_str ? error_str : unknown_str) \ + << " (" \ + << (error_name ? error_name : unknown_name) \ + << ") - "; \ + tf::ostreamize(oss, TF_CUDA_REMOVE_FIRST(__VA_ARGS__)); \ + throw std::runtime_error(oss.str()); \ +} + diff --git a/bundled/cpp-taskflow/include/taskflow/cuda/cuda_flow.hpp b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_flow.hpp new file mode 100644 index 0000000000..efe770a014 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_flow.hpp @@ -0,0 +1,503 @@ +#pragma once + +#include "cuda_task.hpp" + +namespace tf { + +/** +@class cudaFlow + +@brief methods for building a CUDA task dependency graph. + +A cudaFlow is a high-level interface to manipulate GPU tasks using +the task dependency graph model. +The class provides a set of methods for creating and launch different tasks +on one or multiple CUDA devices, +for instance, kernel tasks, data transfer tasks, and memory operation tasks. +*/ +class cudaFlow { + + friend class Executor; + + public: + + /** + @brief constructs a cudaFlow builder object + + @tparam P predicate type + + @param graph a cudaGraph to manipulate + @param p predicate which return @c true if the launching should be contined + */ + template + cudaFlow(cudaGraph& graph, P&& p); + + /** + @brief queries the emptiness of the graph + */ + bool empty() const; + + /** + @brief creates a no-operation task + + An empty node performs no operation during execution, + but can be used for transitive ordering. + For example, a phased execution graph with 2 groups of n nodes + with a barrier between them can be represented using an empty node + and 2*n dependency edges, + rather than no empty node and n^2 dependency edges. + */ + cudaTask noop(); + + // CUDA seems pretty restrictive about calling host in a cudaGraph. + // We disable this function and wait for future stability. + // + //@brief creates a host execution task + // + //@tparam C callable type + // + //@param c a callable object constructible from std::function. + + //A host can only execute CPU-specific functions and cannot do any CUDA calls + //(e.g., cudaMalloc). + // + //template + //cudaTask host(C&& c); + + /** + @brief creates a kernel task + + @tparam F kernel function type + @tparam ArgsT kernel function parameters type + + @param g configured grid + @param b configured block + @param s configured shared memory + @param f kernel function + @param args arguments to forward to the kernel function by copy + + @return cudaTask handle + */ + template + cudaTask kernel(dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args); + + /** + @brief creates a kernel task on a device + + @tparam F kernel function type + @tparam ArgsT kernel function parameters type + + @param d device identifier to luanch the kernel + @param g configured grid + @param b configured block + @param s configured shared memory + @param f kernel function + @param args arguments to forward to the kernel function by copy + + @return cudaTask handle + */ + template + cudaTask kernel_on(int d, dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args); + + /** + @brief creates a memset task + + @param dst pointer to the destination device memory area + @param v value to set for each byte of specified memory + @param count size in bytes to set + + A memset task fills the first @c count bytes of device memory area + pointed by @c dst with the byte value @c v. + */ + cudaTask memset(void* dst, int v, size_t count); + + /** + @brief creates a memcpy task + + @param tgt pointer to the target memory block + @param src pointer to the source memory block + @param bytes bytes to copy + + @return cudaTask handle + + A memcpy task transfers @c bytes of data from a course location + to a target location. Direction can be arbitrary among CPUs and GPUs. + */ + cudaTask memcpy(void* tgt, const void* src, size_t bytes); + + /** + @brief creates a zero task that zeroes a typed memory block + + @tparam T element type (size of @c T must be either 1, 2, or 4) + @param dst pointer to the destination device memory area + @param count number of elements + + A zero task zeroes the first @c count elements of type @c T + in a device memory area pointed by @c dst. + */ + template + std::enable_if_t< + is_pod_v && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), + cudaTask + > + zero(T* dst, size_t count); + + /** + @brief creates a fill task that fills a typed memory block with a value + + @tparam T element type (size of @c T must be either 1, 2, or 4) + @param dst pointer to the destination device memory area + @param value value to fill for each element of type @c T + @param count number of elements + + A fill task fills the first @c count elements of type @c T with @c value + in a device memory area pointed by @c dst. + The value to fill is interpreted in type @c T rather than byte. + */ + template + std::enable_if_t< + is_pod_v && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), + cudaTask + > + fill(T* dst, T value, size_t count); + + /** + @brief creates a copy task + + @tparam T element type (non-void) + + @param tgt pointer to the target memory block + @param src pointer to the source memory block + @param num number of elements to copy + + @return cudaTask handle + + A copy task transfers num*sizeof(T) bytes of data from a source location + to a target location. Direction can be arbitrary among CPUs and GPUs. + */ + template < + typename T, + std::enable_if_t::value, void>* = nullptr + > + cudaTask copy(T* tgt, const T* src, size_t num); + + /** + @brief assigns a device to launch the cudaFlow + + @param device target device identifier + */ + void device(int device); + + /** + @brief queries the device associated with the cudaFlow + */ + int device() const; + + /** + @brief assigns a stream to launch the cudaFlow + + @param stream target stream identifier + */ + void stream(cudaStream_t stream); + + /** + @brief assigns a predicate to loop the cudaFlow until the predicate is satisfied + + @tparam P predicate type + @param p predicate which return @c true if the launching should be contined + + The execution of cudaFlow is equivalent to: while(!predicate()) { run cudaflow; } + */ + template + void predicate(P&& p); + + /** + @brief repeats the execution of the cudaFlow by @c n times + */ + void repeat(size_t n); + + private: + + cudaGraph& _graph; + + int _device {0}; + + nstd::optional _stream; + + std::function _predicate; +}; + +// Constructor +template +cudaFlow::cudaFlow(cudaGraph& g, P&& p) : + _graph {g}, + _predicate {std::forward

(p)} { +} + +// Procedure: predicate +template +void cudaFlow::predicate(P&& pred) { + _predicate = std::forward

(pred); +} + +// Procedure: repeat +inline void cudaFlow::repeat(size_t n) { + _predicate = [n] () mutable { return n-- == 0; }; +} + +// Function: empty +inline bool cudaFlow::empty() const { + return _graph._nodes.empty(); +} + +// Procedure: device +inline void cudaFlow::device(int d) { + _device = d; +} + +// Function: device +inline int cudaFlow::device() const { + return _device; +} + +// Procedure: stream +inline void cudaFlow::stream(cudaStream_t s) { + _stream = s; +} + +// Function: noop +inline cudaTask cudaFlow::noop() { + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [](cudaGraph_t& graph, cudaGraphNode_t& node){ + TF_CHECK_CUDA( + ::cudaGraphAddEmptyNode(&node, graph, nullptr, 0), + "failed to create a no-operation (empty) node" + ); + } + ); + return cudaTask(node); +} + +//// Function: host +//template +//cudaTask cudaFlow::host(C&& c) { +// auto node = _graph.emplace_back(nstd::in_place_type_t{}, +// [c=std::forward(c)](cudaGraph_t& graph, cudaGraphNode_t& node) mutable { +// cudaHostNodeParams p; +// p.fn = [] (void* data) { (*static_cast(data))(); }; +// p.userData = &c; +// TF_CHECK_CUDA( +// ::cudaGraphAddHostNode(&node, graph, nullptr, 0, &p), +// "failed to create a host node" +// ); +// } +// ); +// return cudaTask(node); +//} + +// Function: kernel +template +cudaTask cudaFlow::kernel( + dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args +) { + + using traits = function_traits; + + static_assert(traits::arity == sizeof...(ArgsT), "arity mismatches"); + + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [g, b, s, f=(void*)f, args...] (cudaGraph_t& graph, cudaGraphNode_t& node) { + + cudaKernelNodeParams p; + void* arguments[sizeof...(ArgsT)] = { (void*)(&args)... }; + p.func = f; + p.gridDim = g; + p.blockDim = b; + p.sharedMemBytes = s; + p.kernelParams = arguments; + p.extra = nullptr; + + TF_CHECK_CUDA( + ::cudaGraphAddKernelNode(&node, graph, nullptr, 0, &p), + "failed to create a cudaGraph node in kernel task" + ); + } + ); + + return cudaTask(node); +} + +// Function: kernel +template +cudaTask cudaFlow::kernel_on( + int d, dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args +) { + + using traits = function_traits; + + static_assert(traits::arity == sizeof...(ArgsT), "arity mismatches"); + + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [d, g, b, s, f=(void*)f, args...] (cudaGraph_t& graph, cudaGraphNode_t& node) { + + cudaKernelNodeParams p; + void* arguments[sizeof...(ArgsT)] = { (void*)(&args)... }; + p.func = f; + p.gridDim = g; + p.blockDim = b; + p.sharedMemBytes = s; + p.kernelParams = arguments; + p.extra = nullptr; + + cudaScopedDevice ctx(d); + TF_CHECK_CUDA( + ::cudaGraphAddKernelNode(&node, graph, nullptr, 0, &p), + "failed to create a cudaGraph node in kernel_on task" + ); + } + ); + + return cudaTask(node); +} + +// Function: zero +template +std::enable_if_t< + is_pod_v && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), + cudaTask +> +cudaFlow::zero(T* dst, size_t count) { + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [dst, count] (cudaGraph_t& graph, cudaGraphNode_t& node) { + cudaMemsetParams p; + p.dst = dst; + p.value = 0; + p.pitch = 0; + p.elementSize = sizeof(T); // either 1, 2, or 4 + p.width = count; + p.height = 1; + TF_CHECK_CUDA( + cudaGraphAddMemsetNode(&node, graph, nullptr, 0, &p), + "failed to create a cudaGraph node in zero task" + ); + } + ); + return cudaTask(node); +} + +// Function: fill +template +std::enable_if_t< + is_pod_v && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), + cudaTask +> +cudaFlow::fill(T* dst, T value, size_t count) { + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [dst, value, count] (cudaGraph_t& graph, cudaGraphNode_t& node) { + cudaMemsetParams p; + p.dst = dst; + + // perform bit-wise copy + p.value = 0; // crucial + static_assert(sizeof(T) <= sizeof(p.value), "internal error"); + std::memcpy(&p.value, &value, sizeof(T)); + + p.pitch = 0; + p.elementSize = sizeof(T); // either 1, 2, or 4 + p.width = count; + p.height = 1; + TF_CHECK_CUDA( + cudaGraphAddMemsetNode(&node, graph, nullptr, 0, &p), + "failed to create a cudaGraph node in fill task" + ); + } + ); + return cudaTask(node); +} + +// Function: copy +template < + typename T, + std::enable_if_t::value, void>* +> +cudaTask cudaFlow::copy(T* tgt, const T* src, size_t num) { + + using U = std::decay_t; + + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [tgt, src, num] (cudaGraph_t& graph, cudaGraphNode_t& node) { + + cudaMemcpy3DParms p; + p.srcArray = nullptr; + p.srcPos = ::make_cudaPos(0, 0, 0); + p.srcPtr = ::make_cudaPitchedPtr(const_cast(src), num*sizeof(U), num, 1); + p.dstArray = nullptr; + p.dstPos = ::make_cudaPos(0, 0, 0); + p.dstPtr = ::make_cudaPitchedPtr(tgt, num*sizeof(U), num, 1); + p.extent = ::make_cudaExtent(num*sizeof(U), 1, 1); + p.kind = cudaMemcpyDefault; + + TF_CHECK_CUDA( + cudaGraphAddMemcpyNode(&node, graph, nullptr, 0, &p), + "failed to create a cudaGraph node in copy task" + ); + } + ); + + return cudaTask(node); +} + +// Function: memset +inline cudaTask cudaFlow::memset(void* dst, int ch, size_t count) { + + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [dst, ch, count] (cudaGraph_t& graph, cudaGraphNode_t& node) { + cudaMemsetParams p; + p.dst = dst; + p.value = ch; + p.pitch = 0; + //p.elementSize = (count & 1) == 0 ? ((count & 3) == 0 ? 4 : 2) : 1; + //p.width = (count & 1) == 0 ? ((count & 3) == 0 ? count >> 2 : count >> 1) : count; + p.elementSize = 1; // either 1, 2, or 4 + p.width = count; + p.height = 1; + TF_CHECK_CUDA( + cudaGraphAddMemsetNode(&node, graph, nullptr, 0, &p), + "failed to create a cudaGraph node in memset task" + ); + } + ); + + return cudaTask(node); +} + +// Function: memcpy +inline cudaTask cudaFlow::memcpy(void* tgt, const void* src, size_t bytes) { + auto node = _graph.emplace_back(nstd::in_place_type_t{}, + [tgt, src, bytes] (cudaGraph_t& graph, cudaGraphNode_t& node) { + // Parameters in cudaPitchedPtr + // d - Pointer to allocated memory + // p - Pitch of allocated memory in bytes + // xsz - Logical width of allocation in elements + // ysz - Logical height of allocation in elements + cudaMemcpy3DParms p; + p.srcArray = nullptr; + p.srcPos = ::make_cudaPos(0, 0, 0); + p.srcPtr = ::make_cudaPitchedPtr(const_cast(src), bytes, bytes, 1); + p.dstArray = nullptr; + p.dstPos = ::make_cudaPos(0, 0, 0); + p.dstPtr = ::make_cudaPitchedPtr(tgt, bytes, bytes, 1); + p.extent = ::make_cudaExtent(bytes, 1, 1); + p.kind = cudaMemcpyDefault; + TF_CHECK_CUDA( + cudaGraphAddMemcpyNode(&node, graph, nullptr, 0, &p), + "failed to create a cudaGraph node in memcpy task" + ); + } + ); + return cudaTask(node); +} + +} // end of namespace tf ----------------------------------------------------- + + diff --git a/bundled/cpp-taskflow/include/taskflow/cuda/cuda_graph.hpp b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_graph.hpp new file mode 100644 index 0000000000..cb42617711 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_graph.hpp @@ -0,0 +1,310 @@ +#pragma once + +#include "cuda_device.hpp" + +#include "../utility/object_pool.hpp" +#include "../utility/traits.hpp" +#include "../utility/passive_vector.hpp" +#include "../nstd/variant.hpp" +#include "../nstd/optional.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- +// cudaNode class +// ---------------------------------------------------------------------------- + +// class: cudaNode +class cudaNode { + + friend class cudaFlow; + friend class cudaGraph; + friend class cudaTask; + + friend class Taskflow; + friend class Executor; + + + // Noop handle + struct Noop { + + template + Noop(C&&); + + std::function work; + }; + + //// Host handle + //struct Host { + + // template + // Host(C&&); + // + // std::function work; + //}; + + // Memset handle + struct Memset { + + template + Memset(C&&); + + std::function work; + }; + + // Copy handle + struct Copy { + + template + Copy(C&&); + + std::function work; + }; + + // Kernel handle + struct Kernel { + + template + Kernel(C&&); + + std::function work; + }; + + using handle_t = nstd::variant< + nstd::monostate, + Noop, + //Host, + Memset, + Copy, + Kernel + >; + + // variant index + constexpr static auto NOOP = get_index_v; + //constexpr static auto HOST = get_index_v; + constexpr static auto MEMSET = get_index_v; + constexpr static auto COPY = get_index_v; + constexpr static auto KERNEL = get_index_v; + + public: + + template + cudaNode(ArgsT&&...); + + private: + + std::string _name; + + handle_t _handle; + + cudaGraphNode_t _native_handle {nullptr}; + + PassiveVector _successors; + + void _precede(cudaNode*); +}; + +// ---------------------------------------------------------------------------- +// cudaGraph class +// ---------------------------------------------------------------------------- + +// class: cudaGraph +class cudaGraph { + + friend class cudaFlow; + friend class cudaNode; + friend class cudaTask; + + friend class Taskflow; + friend class Executor; + + public: + + ~cudaGraph(); + + template + cudaNode* emplace_back(ArgsT&&...); + + cudaGraph_t native_handle(); + + void clear(); + + bool empty() const; + + private: + + cudaGraph_t _native_handle {nullptr}; + + std::vector> _nodes; + + void _make_native_graph(); +}; + +// ---------------------------------------------------------------------------- +// cudaNode definitions +// ---------------------------------------------------------------------------- + +//// Host handle constructor +//template +//cudaNode::Host::Host(C&& c) : work {std::forward(c)} { +//} + +// Noop handle constructor +template +cudaNode::Noop::Noop(C&& c) : work {std::forward(c)} { +} + +// Memset handle constructor +template +cudaNode::Memset::Memset(C&& c) : work {std::forward(c)} { +} + +// Copy handle constructor +template +cudaNode::Copy::Copy(C&& c) : work {std::forward(c)} { +} + +// Kernel handle constructor +template +cudaNode::Kernel::Kernel(C&& c) : work {std::forward(c)} { +} + +// Constructor +template +cudaNode::cudaNode(ArgsT&&... args) : _handle {std::forward(args)...} { +} + +// Procedure: _precede +inline void cudaNode::_precede(cudaNode* v) { + _successors.push_back(v); +} + +// ---------------------------------------------------------------------------- +// cudaGraph definitions +// ---------------------------------------------------------------------------- + +// Destructor +inline cudaGraph::~cudaGraph() { + if(_native_handle) { + cudaGraphDestroy(_native_handle); + } +} + +// Function: empty +inline bool cudaGraph::empty() const { + return _nodes.empty(); +} + +// Procedure: clear +inline void cudaGraph::clear() { + + _nodes.clear(); + + if(_native_handle) { + TF_CHECK_CUDA( + cudaGraphDestroy(_native_handle), "failed to destroy a cudaGraph on clear" + ); + _native_handle = nullptr; + } +} + +// Function: emplace_back +template +cudaNode* cudaGraph::emplace_back(ArgsT&&... args) { + auto node = std::make_unique(std::forward(args)...); + _nodes.emplace_back(std::move(node)); + return _nodes.back().get(); +} + +// Function: native_handle +inline cudaGraph_t cudaGraph::native_handle() { + return _native_handle; +} + +// Procedure: _make_native_graph +inline void cudaGraph::_make_native_graph() { + + //// TODO: must be nullptr + //if(_native_handle) { + // TF_CHECK_CUDA( + // cudaGraphDestroy(_native_handle), "failed to destroy the previous cudaGraph" + // ); + // _native_handle = nullptr; + //} + // + //cudaScopedDevice ctx {d}; + assert(_native_handle == nullptr); + + TF_CHECK_CUDA( + cudaGraphCreate(&_native_handle, 0), "failed to create a cudaGraph" + ); + + // create nodes + for(auto& node : _nodes) { + switch(node->_handle.index()) { + case cudaNode::NOOP: + nstd::get(node->_handle).work( + _native_handle, node->_native_handle + ); + break; + + //case cudaNode::HOST: + // nstd::get(node->_handle).work( + // _native_handle, node->_native_handle + // ); + //break; + + case cudaNode::MEMSET: + nstd::get(node->_handle).work( + _native_handle, node->_native_handle + ); + break; + + case cudaNode::COPY: + nstd::get(node->_handle).work( + _native_handle, node->_native_handle + ); + break; + + case cudaNode::KERNEL: + nstd::get(node->_handle).work( + _native_handle, node->_native_handle + ); + break; + } + } + + // create edges + for(auto& node : _nodes) { + for(auto succ : node->_successors){ + TF_CHECK_CUDA( + ::cudaGraphAddDependencies( + _native_handle, &(node->_native_handle), &(succ->_native_handle), 1 + ), + "failed to add a preceding link" + ); + } + } + +} + + +//inline void cudaGraph::run() { +// cudaGraphExec_t graphExec; +// TF_CHECK_CUDA( +// cudaGraphInstantiate(&graphExec, _handle, nullptr, nullptr, 0), +// "failed to create an executable cudaGraph" +// ); +// TF_CHECK_CUDA(cudaGraphLaunch(graphExec, 0), "failed to launch cudaGraph") +// TF_CHECK_CUDA(cudaStreamSynchronize(0), "failed to sync cudaStream"); +// TF_CHECK_CUDA( +// cudaGraphExecDestroy(graphExec), "failed to destroy an executable cudaGraph" +// ); +//} + + + + + +} // end of namespace tf ----------------------------------------------------- + diff --git a/bundled/cpp-taskflow/include/taskflow/cuda/cuda_task.hpp b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_task.hpp new file mode 100644 index 0000000000..670765f786 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/cuda/cuda_task.hpp @@ -0,0 +1,173 @@ +#pragma once + +#include "cuda_graph.hpp" + +namespace tf { + +/** +@class cudaTask + +@brief handle to a node in a cudaGraph +*/ +class cudaTask { + + friend class cudaFlow; + + public: + + /** + @brief constructs an empty cudaTask + */ + cudaTask() = default; + + /** + @brief copy-constructs a cudaTask + */ + cudaTask(const cudaTask&) = default; + + /** + @brief copy-assigns a cudaTask + */ + cudaTask& operator = (const cudaTask&) = default; + + /** + @brief adds precedence links from this to other tasks + + @tparam Ts... parameter pack + + @param tasks one or multiple tasks + + @return @c *this + */ + template + cudaTask& precede(Ts&&... tasks); + + /** + @brief adds precedence links from other tasks to this + + @tparam Ts... parameter pack + + @param tasks one or multiple tasks + + @return @c *this + */ + template + cudaTask& succeed(Ts&&... tasks); + + /** + @brief assigns a name to the task + + @param name a @std_string acceptable string + + @return @c *this + */ + cudaTask& name(const std::string& name); + + /** + @brief queries the name of the task + */ + const std::string& name() const; + + /** + @brief queries the number of successors + */ + size_t num_successors() const; + + /** + @brief queries if the task is associated with a cudaNode + */ + bool empty() const; + + private: + + cudaTask(cudaNode*); + + cudaNode* _node {nullptr}; + + /// @private + template + void _precede(T&&); + + /// @private + template + void _precede(T&&, Ts&&...); + + /// @private + template + void _succeed(T&&); + + // @private + template + void _succeed(T&&, Ts&&...); +}; + +// Constructor +inline cudaTask::cudaTask(cudaNode* node) : _node {node} { +} + +// Function: precede +template +cudaTask& cudaTask::precede(Ts&&... tasks) { + _precede(std::forward(tasks)...); + return *this; +} + +/// @private +// Procedure: precede +template +void cudaTask::_precede(T&& other) { + _node->_precede(other._node); +} + +/// @private +// Procedure: _precede +template +void cudaTask::_precede(T&& task, Ts&&... others) { + _precede(std::forward(task)); + _precede(std::forward(others)...); +} + +// Function: succeed +template +cudaTask& cudaTask::succeed(Ts&&... tasks) { + _succeed(std::forward(tasks)...); + return *this; +} + +/// @private +// Procedure: _succeed +template +void cudaTask::_succeed(T&& other) { + other._node->_precede(_node); +} + +/// @private +// Procedure: _succeed +template +void cudaTask::_succeed(T&& task, Ts&&... others) { + _succeed(std::forward(task)); + _succeed(std::forward(others)...); +} + +// Function: empty +inline bool cudaTask::empty() const { + return _node == nullptr; +} + +// Function: name +inline cudaTask& cudaTask::name(const std::string& name) { + _node->_name = name; + return *this; +} + +// Function: name +inline const std::string& cudaTask::name() const { + return _node->_name; +} + +// Function: num_successors +inline size_t cudaTask::num_successors() const { + return _node->_successors.size(); +} + +} // end of namespace tf ----------------------------------------------------- diff --git a/bundled/cpp-taskflow/include/taskflow/declarations.hpp b/bundled/cpp-taskflow/include/taskflow/declarations.hpp new file mode 100644 index 0000000000..3047b2f9ef --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/declarations.hpp @@ -0,0 +1,36 @@ +#pragma once + +namespace tf { + +// ---------------------------------------------------------------------------- +// forward declarations +// ---------------------------------------------------------------------------- + +// taskflow +class Node; +class Graph; +class FlowBuilder; +class Subflow; +class Task; +class TaskView; +class Taskflow; +class Topology; +class Executor; +class WorkerView; +class ObserverInterface; +class ChromeTracingObserver; +class TFProfObserver; + +// cudaflow +class cudaNode; +class cudaGraph; +class cudaTask; +class cudaFlow; + + + +} // end of namespace tf ----------------------------------------------------- + + + + diff --git a/bundled/cpp-taskflow/include/taskflow/nstd/any.hpp b/bundled/cpp-taskflow/include/taskflow/nstd/any.hpp new file mode 100644 index 0000000000..a27000fe0a --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/nstd/any.hpp @@ -0,0 +1,703 @@ +// +// Copyright (c) 2016-2018 Martin Moene +// +// https://github.com/martinmoene/any-lite +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#ifndef NSTD_ANY_LITE_HPP +#define NSTD_ANY_LITE_HPP + +#define any_lite_MAJOR 0 +#define any_lite_MINOR 2 +#define any_lite_PATCH 0 + +#define any_lite_VERSION any_STRINGIFY(any_lite_MAJOR) "." any_STRINGIFY(any_lite_MINOR) "." any_STRINGIFY(any_lite_PATCH) + +#define any_STRINGIFY( x ) any_STRINGIFY_( x ) +#define any_STRINGIFY_( x ) #x + +// any-lite configuration: + +#define any_ANY_DEFAULT 0 +#define any_ANY_NSTD 1 +#define any_ANY_STD 2 + +#if !defined( any_CONFIG_SELECT_ANY ) +# define any_CONFIG_SELECT_ANY ( any_HAVE_STD_ANY ? any_ANY_STD : any_ANY_NSTD ) +#endif + +// Control presence of exception handling (try and auto discover): + +#ifndef any_CONFIG_NO_EXCEPTIONS +# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +# define any_CONFIG_NO_EXCEPTIONS 0 +# else +# define any_CONFIG_NO_EXCEPTIONS 1 +# endif +#endif + +// C++ language version detection (C++20 is speculative): +// Note: VC14.0/1900 (VS2015) lacks too much from C++14. + +#ifndef any_CPLUSPLUS +# if defined(_MSVC_LANG ) && !defined(__clang__) +# define any_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG ) +# else +# define any_CPLUSPLUS __cplusplus +# endif +#endif + +#define any_CPP98_OR_GREATER ( any_CPLUSPLUS >= 199711L ) +#define any_CPP11_OR_GREATER ( any_CPLUSPLUS >= 201103L ) +#define any_CPP14_OR_GREATER ( any_CPLUSPLUS >= 201402L ) +#define any_CPP17_OR_GREATER ( any_CPLUSPLUS >= 201703L ) +#define any_CPP20_OR_GREATER ( any_CPLUSPLUS >= 202000L ) + +// Use C++17 std::any if available and requested: + +#if any_CPP17_OR_GREATER && defined(__has_include ) +# if __has_include( ) +# define any_HAVE_STD_ANY 1 +# else +# define any_HAVE_STD_ANY 0 +# endif +#else +# define any_HAVE_STD_ANY 0 +#endif + +#define any_USES_STD_ANY ( (any_CONFIG_SELECT_ANY == any_ANY_STD) || ((any_CONFIG_SELECT_ANY == any_ANY_DEFAULT) && any_HAVE_STD_ANY) ) + +// +// in_place: code duplicated in any-lite, expected-lite, optional-lite, value-ptr-lite, variant-lite: +// + +#ifndef nstd_lite_HAVE_IN_PLACE_TYPES +#define nstd_lite_HAVE_IN_PLACE_TYPES 1 + +// C++17 std::in_place in : + +#if any_CPP17_OR_GREATER + +#include + +namespace nstd { + +using std::in_place; +using std::in_place_type; +using std::in_place_index; +using std::in_place_t; +using std::in_place_type_t; +using std::in_place_index_t; + +#define nstd_lite_in_place_t( T) std::in_place_t +#define nstd_lite_in_place_type_t( T) std::in_place_type_t +#define nstd_lite_in_place_index_t(K) std::in_place_index_t + +#define nstd_lite_in_place( T) std::in_place_t{} +#define nstd_lite_in_place_type( T) std::in_place_type_t{} +#define nstd_lite_in_place_index(K) std::in_place_index_t{} + +} // namespace nstd + +#else // any_CPP17_OR_GREATER + +#include + +namespace nstd { +namespace detail { + +template< class T > +struct in_place_type_tag {}; + +template< std::size_t K > +struct in_place_index_tag {}; + +} // namespace detail + +struct in_place_t {}; + +template< class T > +inline in_place_t in_place( detail::in_place_type_tag = detail::in_place_type_tag() ) +{ + return in_place_t(); +} + +template< std::size_t K > +inline in_place_t in_place( detail::in_place_index_tag = detail::in_place_index_tag() ) +{ + return in_place_t(); +} + +template< class T > +inline in_place_t in_place_type( detail::in_place_type_tag = detail::in_place_type_tag() ) +{ + return in_place_t(); +} + +template< std::size_t K > +inline in_place_t in_place_index( detail::in_place_index_tag = detail::in_place_index_tag() ) +{ + return in_place_t(); +} + +// mimic templated typedef: + +#define nstd_lite_in_place_t( T) nstd::in_place_t(&)( nstd::detail::in_place_type_tag ) +#define nstd_lite_in_place_type_t( T) nstd::in_place_t(&)( nstd::detail::in_place_type_tag ) +#define nstd_lite_in_place_index_t(K) nstd::in_place_t(&)( nstd::detail::in_place_index_tag ) + +#define nstd_lite_in_place( T) nstd::in_place_type +#define nstd_lite_in_place_type( T) nstd::in_place_type +#define nstd_lite_in_place_index(K) nstd::in_place_index + +} // namespace nstd + +#endif // any_CPP17_OR_GREATER +#endif // nstd_lite_HAVE_IN_PLACE_TYPES + +// +// Using std::any: +// + +#if any_USES_STD_ANY + +#include +#include + +namespace nstd { + + using std::any; + using std::any_cast; + using std::make_any; + using std::swap; + using std::bad_any_cast; +} + +#else // any_USES_STD_ANY + +#include + +// Compiler versions: +// +// MSVC++ 6.0 _MSC_VER == 1200 any_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0) +// MSVC++ 7.0 _MSC_VER == 1300 any_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002) +// MSVC++ 7.1 _MSC_VER == 1310 any_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003) +// MSVC++ 8.0 _MSC_VER == 1400 any_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005) +// MSVC++ 9.0 _MSC_VER == 1500 any_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008) +// MSVC++ 10.0 _MSC_VER == 1600 any_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010) +// MSVC++ 11.0 _MSC_VER == 1700 any_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012) +// MSVC++ 12.0 _MSC_VER == 1800 any_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013) +// MSVC++ 14.0 _MSC_VER == 1900 any_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015) +// MSVC++ 14.1 _MSC_VER >= 1910 any_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017) +// MSVC++ 14.2 _MSC_VER >= 1920 any_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019) + +#if defined(_MSC_VER ) && !defined(__clang__) +# define any_COMPILER_MSVC_VER (_MSC_VER ) +# define any_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * ( 5 + (_MSC_VER < 1900 ) ) ) +#else +# define any_COMPILER_MSVC_VER 0 +# define any_COMPILER_MSVC_VERSION 0 +#endif + +#define any_COMPILER_VERSION( major, minor, patch ) ( 10 * ( 10 * (major) + (minor) ) + (patch) ) + +#if defined(__clang__) +# define any_COMPILER_CLANG_VERSION any_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__) +#else +# define any_COMPILER_CLANG_VERSION 0 +#endif + +#if defined(__GNUC__) && !defined(__clang__) +# define any_COMPILER_GNUC_VERSION any_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#else +# define any_COMPILER_GNUC_VERSION 0 +#endif + +// half-open range [lo..hi): +//#define any_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) ) + +// Presence of language and library features: + +#define any_HAVE( feature ) ( any_HAVE_##feature ) + +#ifdef _HAS_CPP0X +# define any_HAS_CPP0X _HAS_CPP0X +#else +# define any_HAS_CPP0X 0 +#endif + +#define any_CPP11_90 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1500) +#define any_CPP11_100 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1600) +#define any_CPP11_120 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1800) +#define any_CPP11_140 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1900) + +#define any_CPP14_000 (any_CPP14_OR_GREATER) +#define any_CPP17_000 (any_CPP17_OR_GREATER) + +// Presence of C++11 language features: + +#define any_HAVE_CONSTEXPR_11 any_CPP11_140 +#define any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG \ + any_CPP11_120 +#define any_HAVE_INITIALIZER_LIST any_CPP11_120 +#define any_HAVE_NOEXCEPT any_CPP11_140 +#define any_HAVE_NULLPTR any_CPP11_100 +#define any_HAVE_TYPE_TRAITS any_CPP11_90 +#define any_HAVE_STATIC_ASSERT any_CPP11_100 +#define any_HAVE_ADD_CONST any_CPP11_90 +#define any_HAVE_REMOVE_REFERENCE any_CPP11_90 + +#define any_HAVE_TR1_ADD_CONST (!! any_COMPILER_GNUC_VERSION ) +#define any_HAVE_TR1_REMOVE_REFERENCE (!! any_COMPILER_GNUC_VERSION ) +#define any_HAVE_TR1_TYPE_TRAITS (!! any_COMPILER_GNUC_VERSION ) + +// Presence of C++14 language features: + +#define any_HAVE_CONSTEXPR_14 any_CPP14_000 + +// Presence of C++17 language features: + +#define any_HAVE_NODISCARD any_CPP17_000 + +// Presence of C++ language features: + +#if any_HAVE_CONSTEXPR_11 +# define any_constexpr constexpr +#else +# define any_constexpr /*constexpr*/ +#endif + +#if any_HAVE_CONSTEXPR_14 +# define any_constexpr14 constexpr +#else +# define any_constexpr14 /*constexpr*/ +#endif + +#if any_HAVE_NOEXCEPT +# define any_noexcept noexcept +#else +# define any_noexcept /*noexcept*/ +#endif + +#if any_HAVE_NULLPTR +# define any_nullptr nullptr +#else +# define any_nullptr NULL +#endif + +#if any_HAVE_NODISCARD +# define any_nodiscard [[nodiscard]] +#else +# define any_nodiscard /*[[nodiscard]]*/ +#endif + +// additional includes: + +#if any_CONFIG_NO_EXCEPTIONS +# include +#else +# include +#endif + +#if ! any_HAVE_NULLPTR +# include +#endif + +#if any_HAVE_INITIALIZER_LIST +# include +#endif + +#if any_HAVE_TYPE_TRAITS +# include +#elif any_HAVE_TR1_TYPE_TRAITS +# include +#endif + +// Method enabling + +#if any_CPP11_OR_GREATER + +#define any_REQUIRES_0(...) \ + template< bool B = (__VA_ARGS__), typename std::enable_if::type = 0 > + +#define any_REQUIRES_T(...) \ + , typename = typename std::enable_if< (__VA_ARGS__), nstd::any_lite::detail::enabler >::type + +#define any_REQUIRES_R(R, ...) \ + typename std::enable_if<__VA_ARGS__, R>::type + +#define any_REQUIRES_A(...) \ + , typename std::enable_if<__VA_ARGS__, void*>::type = nullptr + +#endif + +// +// any: +// + +namespace nstd { namespace any_lite { + +// C++11 emulation: + +namespace std11 { + +#if any_HAVE_ADD_CONST + +using std::add_const; + +#elif any_HAVE_TR1_ADD_CONST + +using std::tr1::add_const; + +#else + +template< class T > struct add_const { typedef const T type; }; + +#endif // any_HAVE_ADD_CONST + +#if any_HAVE_REMOVE_REFERENCE + +using std::remove_reference; + +#elif any_HAVE_TR1_REMOVE_REFERENCE + +using std::tr1::remove_reference; + +#else + +template< class T > struct remove_reference { typedef T type; }; +template< class T > struct remove_reference { typedef T type; }; + +#endif // any_HAVE_REMOVE_REFERENCE + +} // namespace std11 + +namespace detail { + +// for any_REQUIRES_T + +/*enum*/ class enabler{}; + +} // namespace detail + +#if ! any_CONFIG_NO_EXCEPTIONS + +class bad_any_cast : public std::bad_cast +{ +public: +#if any_CPP11_OR_GREATER + virtual const char* what() const any_noexcept +#else + virtual const char* what() const throw() +#endif + { + return "any-lite: bad any_cast"; + } +}; + +#endif // any_CONFIG_NO_EXCEPTIONS + +class any +{ +public: + any_constexpr any() any_noexcept + : content( any_nullptr ) + {} + + any( any const & other ) + : content( other.content ? other.content->clone() : any_nullptr ) + {} + +#if any_CPP11_OR_GREATER + + any( any && other ) any_noexcept + : content( std::move( other.content ) ) + { + other.content = any_nullptr; + } + + template< + class ValueType, class T = typename std::decay::type + any_REQUIRES_T( ! std::is_same::value ) + > + any( ValueType && value ) any_noexcept + : content( new holder( std::forward( value ) ) ) + {} + + template< + class T, class... Args + any_REQUIRES_T( std::is_constructible::value ) + > + explicit any( nstd_lite_in_place_type_t(T), Args&&... args ) + : content( new holder( T( std::forward(args)... ) ) ) + {} + + template< + class T, class U, class... Args + any_REQUIRES_T( std::is_constructible&, Args&&...>::value ) + > + explicit any( nstd_lite_in_place_type_t(T), std::initializer_list il, Args&&... args ) + : content( new holder( T( il, std::forward(args)... ) ) ) + {} + +#else + + template< class ValueType > + any( ValueType const & value ) + : content( new holder( value ) ) + {} + +#endif // any_CPP11_OR_GREATER + + ~any() + { + reset(); + } + + any & operator=( any const & other ) + { + any( other ).swap( *this ); + return *this; + } + +#if any_CPP11_OR_GREATER + + any & operator=( any && other ) any_noexcept + { + any( std::move( other ) ).swap( *this ); + return *this; + } + + template< + class ValueType, class T = typename std::decay::type + any_REQUIRES_T( ! std::is_same::value ) + > + any & operator=( ValueType && value ) + { + any( std::move( value ) ).swap( *this ); + return *this; + } + + template< class T, class... Args > + void emplace( Args && ... args ) + { + any( T( std::forward(args)... ) ).swap( *this ); + } + + template< + class T, class U, class... Args + any_REQUIRES_T( std::is_constructible&, Args&&...>::value ) + > + void emplace( std::initializer_list il, Args&&... args ) + { + any( T( il, std::forward(args)... ) ).swap( *this ); + } + +#else + + template< class ValueType > + any & operator=( ValueType const & value ) + { + any( value ).swap( *this ); + return *this; + } + +#endif // any_CPP11_OR_GREATER + + void reset() any_noexcept + { + delete content; content = any_nullptr; + } + + void swap( any & other ) any_noexcept + { + std::swap( content, other.content ); + } + + bool has_value() const any_noexcept + { + return content != any_nullptr; + } + + const std::type_info & type() const any_noexcept + { + return has_value() ? content->type() : typeid( void ); + } + + // + // non-standard: + // + + template< class ValueType > + const ValueType * to_ptr() const + { + return &( static_cast *>( content )->held ); + } + + template< class ValueType > + ValueType * to_ptr() + { + return &( static_cast *>( content )->held ); + } + +private: + class placeholder + { + public: + virtual ~placeholder() + { + } + + virtual std::type_info const & type() const = 0; + + virtual placeholder * clone() const = 0; + }; + + template< typename ValueType > + class holder : public placeholder + { + public: + holder( ValueType const & value ) + : held( value ) + {} + +#if any_CPP11_OR_GREATER + holder( ValueType && value ) + : held( std::move( value ) ) + {} +#endif + + virtual std::type_info const & type() const + { + return typeid( ValueType ); + } + + virtual placeholder * clone() const + { + return new holder( held ); + } + + ValueType held; + }; + + placeholder * content; +}; + +inline void swap( any & x, any & y ) any_noexcept +{ + x.swap( y ); +} + +#if any_CPP11_OR_GREATER + +template< class T, class ...Args > +inline any make_any( Args&& ...args ) +{ + return any( nstd_lite_in_place_type(T), std::forward(args)...); +} + +template< class T, class U, class ...Args > +inline any make_any( std::initializer_list il, Args&& ...args ) +{ + return any( nstd_lite_in_place_type(T), il, std::forward(args)...); +} + +#endif // any_CPP11_OR_GREATER + +template< + class ValueType +#if any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG +// any_REQUIRES_T(...) Allow for VC120 (VS2013): + , typename = typename std::enable_if< (std::is_reference::value || std::is_copy_constructible::value), nstd::any_lite::detail::enabler >::type +#endif +> +any_nodiscard inline ValueType any_cast( any const & operand ) +{ + const ValueType * result = any_cast< typename std11::add_const< typename std11::remove_reference::type >::type >( &operand ); + +#if any_CONFIG_NO_EXCEPTIONS + assert( result ); +#else + if ( ! result ) + { + throw bad_any_cast(); + } +#endif + + return *result; +} + +template< + class ValueType +#if any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG +// any_REQUIRES_T(...) Allow for VC120 (VS2013): + , typename = typename std::enable_if< (std::is_reference::value || std::is_copy_constructible::value), nstd::any_lite::detail::enabler >::type +#endif +> +any_nodiscard inline ValueType any_cast( any & operand ) +{ + const ValueType * result = any_cast< typename std11::remove_reference::type >( &operand ); + +#if any_CONFIG_NO_EXCEPTIONS + assert( result ); +#else + if ( ! result ) + { + throw bad_any_cast(); + } +#endif + + return *result; +} + +#if any_CPP11_OR_GREATER + +template< + class ValueType +#if any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG + any_REQUIRES_T( std::is_reference::value || std::is_copy_constructible::value ) +#endif +> +any_nodiscard inline ValueType any_cast( any && operand ) +{ + const ValueType * result = any_cast< typename std11::remove_reference::type >( &operand ); + +#if any_CONFIG_NO_EXCEPTIONS + assert( result ); +#else + if ( ! result ) + { + throw bad_any_cast(); + } +#endif + + return *result; +} + +#endif // any_CPP11_OR_GREATER + +template< class ValueType > +any_nodiscard inline ValueType const * any_cast( any const * operand ) any_noexcept +{ + return operand != any_nullptr && operand->type() == typeid(ValueType) ? operand->to_ptr() : any_nullptr; +} + +template +any_nodiscard inline ValueType * any_cast( any * operand ) any_noexcept +{ + return operand != any_nullptr && operand->type() == typeid(ValueType) ? operand->to_ptr() : any_nullptr; +} + +} // namespace any_lite + +using namespace any_lite; + +} // namespace nstd + +#endif // any_USES_STD_ANY + +#endif // NSTD_ANY_LITE_HPP diff --git a/bundled/cpp-taskflow/include/taskflow/nstd/optional.hpp b/bundled/cpp-taskflow/include/taskflow/nstd/optional.hpp new file mode 100644 index 0000000000..578ea03d93 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/nstd/optional.hpp @@ -0,0 +1,1718 @@ +// +// Copyright (c) 2014-2018 Martin Moene +// +// https://github.com/martinmoene/optional-lite +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#define optional_lite_MAJOR 3 +#define optional_lite_MINOR 2 +#define optional_lite_PATCH 0 + +#define optional_lite_VERSION optional_STRINGIFY(optional_lite_MAJOR) "." optional_STRINGIFY(optional_lite_MINOR) "." optional_STRINGIFY(optional_lite_PATCH) + +#define optional_STRINGIFY( x ) optional_STRINGIFY_( x ) +#define optional_STRINGIFY_( x ) #x + +// optional-lite configuration: + +#define optional_OPTIONAL_DEFAULT 0 +#define optional_OPTIONAL_NONSTD 1 +#define optional_OPTIONAL_STD 2 + +#if !defined( optional_CONFIG_SELECT_OPTIONAL ) +# define optional_CONFIG_SELECT_OPTIONAL ( optional_HAVE_STD_OPTIONAL ? optional_OPTIONAL_STD : optional_OPTIONAL_NONSTD ) +#endif + +// Control presence of exception handling (try and auto discover): + +#ifndef optional_CONFIG_NO_EXCEPTIONS +# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +# define optional_CONFIG_NO_EXCEPTIONS 0 +# else +# define optional_CONFIG_NO_EXCEPTIONS 1 +# endif +#endif + +// C++ language version detection (C++20 is speculative): +// Note: VC14.0/1900 (VS2015) lacks too much from C++14. + +#ifndef optional_CPLUSPLUS +# if defined(_MSVC_LANG ) && !defined(__clang__) +# define optional_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG ) +# else +# define optional_CPLUSPLUS __cplusplus +# endif +#endif + +#define optional_CPP98_OR_GREATER ( optional_CPLUSPLUS >= 199711L ) +#define optional_CPP11_OR_GREATER ( optional_CPLUSPLUS >= 201103L ) +#define optional_CPP11_OR_GREATER_ ( optional_CPLUSPLUS >= 201103L ) +#define optional_CPP14_OR_GREATER ( optional_CPLUSPLUS >= 201402L ) +#define optional_CPP17_OR_GREATER ( optional_CPLUSPLUS >= 201703L ) +#define optional_CPP20_OR_GREATER ( optional_CPLUSPLUS >= 202000L ) + +// C++ language version (represent 98 as 3): + +#define optional_CPLUSPLUS_V ( optional_CPLUSPLUS / 100 - (optional_CPLUSPLUS > 200000 ? 2000 : 1994) ) + +// Use C++17 std::optional if available and requested: + +#if optional_CPP17_OR_GREATER && defined(__has_include ) +# if __has_include( ) +# define optional_HAVE_STD_OPTIONAL 1 +# else +# define optional_HAVE_STD_OPTIONAL 0 +# endif +#else +# define optional_HAVE_STD_OPTIONAL 0 +#endif + +#define optional_USES_STD_OPTIONAL ( (optional_CONFIG_SELECT_OPTIONAL == optional_OPTIONAL_STD) || ((optional_CONFIG_SELECT_OPTIONAL == optional_OPTIONAL_DEFAULT) && optional_HAVE_STD_OPTIONAL) ) + +// +// in_place: code duplicated in any-lite, expected-lite, optional-lite, value-ptr-lite, variant-lite: +// + +#ifndef nonstd_lite_HAVE_IN_PLACE_TYPES +#define nonstd_lite_HAVE_IN_PLACE_TYPES 1 + +// C++17 std::in_place in : + +#if optional_CPP17_OR_GREATER + +#include + +namespace nonstd { + +using std::in_place; +using std::in_place_type; +using std::in_place_index; +using std::in_place_t; +using std::in_place_type_t; +using std::in_place_index_t; + +#define nonstd_lite_in_place_t( T) std::in_place_t +#define nonstd_lite_in_place_type_t( T) std::in_place_type_t +#define nonstd_lite_in_place_index_t(K) std::in_place_index_t + +#define nonstd_lite_in_place( T) std::in_place_t{} +#define nonstd_lite_in_place_type( T) std::in_place_type_t{} +#define nonstd_lite_in_place_index(K) std::in_place_index_t{} + +} // namespace nonstd + +#else // optional_CPP17_OR_GREATER + +#include + +namespace nonstd { +namespace detail { + +template< class T > +struct in_place_type_tag {}; + +template< std::size_t K > +struct in_place_index_tag {}; + +} // namespace detail + +struct in_place_t {}; + +template< class T > +inline in_place_t in_place( detail::in_place_type_tag /*unused*/ = detail::in_place_type_tag() ) +{ + return in_place_t(); +} + +template< std::size_t K > +inline in_place_t in_place( detail::in_place_index_tag /*unused*/ = detail::in_place_index_tag() ) +{ + return in_place_t(); +} + +template< class T > +inline in_place_t in_place_type( detail::in_place_type_tag /*unused*/ = detail::in_place_type_tag() ) +{ + return in_place_t(); +} + +template< std::size_t K > +inline in_place_t in_place_index( detail::in_place_index_tag /*unused*/ = detail::in_place_index_tag() ) +{ + return in_place_t(); +} + +// mimic templated typedef: + +#define nonstd_lite_in_place_t( T) nonstd::in_place_t(&)( nonstd::detail::in_place_type_tag ) +#define nonstd_lite_in_place_type_t( T) nonstd::in_place_t(&)( nonstd::detail::in_place_type_tag ) +#define nonstd_lite_in_place_index_t(K) nonstd::in_place_t(&)( nonstd::detail::in_place_index_tag ) + +#define nonstd_lite_in_place( T) nonstd::in_place_type +#define nonstd_lite_in_place_type( T) nonstd::in_place_type +#define nonstd_lite_in_place_index(K) nonstd::in_place_index + +} // namespace nonstd + +#endif // optional_CPP17_OR_GREATER +#endif // nonstd_lite_HAVE_IN_PLACE_TYPES + +// +// Using std::optional: +// + +#if optional_USES_STD_OPTIONAL + +#include + +namespace nonstd { + + using std::optional; + using std::bad_optional_access; + using std::hash; + + using std::nullopt; + using std::nullopt_t; + + using std::operator==; + using std::operator!=; + using std::operator<; + using std::operator<=; + using std::operator>; + using std::operator>=; + using std::make_optional; + using std::swap; +} + +#else // optional_USES_STD_OPTIONAL + +#include +#include + +// optional-lite alignment configuration: + +#ifndef optional_CONFIG_MAX_ALIGN_HACK +# define optional_CONFIG_MAX_ALIGN_HACK 0 +#endif + +#ifndef optional_CONFIG_ALIGN_AS +// no default, used in #if defined() +#endif + +#ifndef optional_CONFIG_ALIGN_AS_FALLBACK +# define optional_CONFIG_ALIGN_AS_FALLBACK double +#endif + +// Compiler warning suppression: + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wundef" +#elif defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wundef" +#elif defined(_MSC_VER ) +# pragma warning( push ) +#endif + +// half-open range [lo..hi): +#define optional_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) ) + +// Compiler versions: +// +// MSVC++ 6.0 _MSC_VER == 1200 optional_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0) +// MSVC++ 7.0 _MSC_VER == 1300 optional_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002) +// MSVC++ 7.1 _MSC_VER == 1310 optional_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003) +// MSVC++ 8.0 _MSC_VER == 1400 optional_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005) +// MSVC++ 9.0 _MSC_VER == 1500 optional_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008) +// MSVC++ 10.0 _MSC_VER == 1600 optional_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010) +// MSVC++ 11.0 _MSC_VER == 1700 optional_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012) +// MSVC++ 12.0 _MSC_VER == 1800 optional_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013) +// MSVC++ 14.0 _MSC_VER == 1900 optional_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015) +// MSVC++ 14.1 _MSC_VER >= 1910 optional_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017) +// MSVC++ 14.2 _MSC_VER >= 1920 optional_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019) + +#if defined(_MSC_VER ) && !defined(__clang__) +# define optional_COMPILER_MSVC_VER (_MSC_VER ) +# define optional_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * ( 5 + (_MSC_VER < 1900 ) ) ) +#else +# define optional_COMPILER_MSVC_VER 0 +# define optional_COMPILER_MSVC_VERSION 0 +#endif + +#define optional_COMPILER_VERSION( major, minor, patch ) ( 10 * (10 * (major) + (minor) ) + (patch) ) + +#if defined(__GNUC__) && !defined(__clang__) +# define optional_COMPILER_GNUC_VERSION optional_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#else +# define optional_COMPILER_GNUC_VERSION 0 +#endif + +#if defined(__clang__) +# define optional_COMPILER_CLANG_VERSION optional_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__) +#else +# define optional_COMPILER_CLANG_VERSION 0 +#endif + +#if optional_BETWEEN(optional_COMPILER_MSVC_VERSION, 70, 140 ) +# pragma warning( disable: 4345 ) // initialization behavior changed +#endif + +#if optional_BETWEEN(optional_COMPILER_MSVC_VERSION, 70, 150 ) +# pragma warning( disable: 4814 ) // in C++14 'constexpr' will not imply 'const' +#endif + +// Presence of language and library features: + +#define optional_HAVE(FEATURE) ( optional_HAVE_##FEATURE ) + +#ifdef _HAS_CPP0X +# define optional_HAS_CPP0X _HAS_CPP0X +#else +# define optional_HAS_CPP0X 0 +#endif + +// Unless defined otherwise below, consider VC14 as C++11 for optional-lite: + +#if optional_COMPILER_MSVC_VER >= 1900 +# undef optional_CPP11_OR_GREATER +# define optional_CPP11_OR_GREATER 1 +#endif + +#define optional_CPP11_90 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1500) +#define optional_CPP11_100 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1600) +#define optional_CPP11_110 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1700) +#define optional_CPP11_120 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1800) +#define optional_CPP11_140 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1900) +#define optional_CPP11_141 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1910) + +#define optional_CPP11_140_490 ((optional_CPP11_OR_GREATER_ && optional_COMPILER_GNUC_VERSION >= 490) || (optional_COMPILER_MSVC_VER >= 1910)) + +#define optional_CPP14_000 (optional_CPP14_OR_GREATER) +#define optional_CPP17_000 (optional_CPP17_OR_GREATER) + +// Presence of C++11 language features: + +#define optional_HAVE_CONSTEXPR_11 optional_CPP11_140 +#define optional_HAVE_IS_DEFAULT optional_CPP11_140 +#define optional_HAVE_NOEXCEPT optional_CPP11_140 +#define optional_HAVE_NULLPTR optional_CPP11_100 +#define optional_HAVE_REF_QUALIFIER optional_CPP11_140_490 +#define optional_HAVE_INITIALIZER_LIST optional_CPP11_140 + +// Presence of C++14 language features: + +#define optional_HAVE_CONSTEXPR_14 optional_CPP14_000 + +// Presence of C++17 language features: + +#define optional_HAVE_NODISCARD optional_CPP17_000 + +// Presence of C++ library features: + +#define optional_HAVE_CONDITIONAL optional_CPP11_120 +#define optional_HAVE_REMOVE_CV optional_CPP11_120 +#define optional_HAVE_TYPE_TRAITS optional_CPP11_90 + +#define optional_HAVE_TR1_TYPE_TRAITS (!! optional_COMPILER_GNUC_VERSION ) +#define optional_HAVE_TR1_ADD_POINTER (!! optional_COMPILER_GNUC_VERSION ) + +// C++ feature usage: + +#if optional_HAVE( CONSTEXPR_11 ) +# define optional_constexpr constexpr +#else +# define optional_constexpr /*constexpr*/ +#endif + +#if optional_HAVE( IS_DEFAULT ) +# define optional_is_default = default; +#else +# define optional_is_default {} +#endif + +#if optional_HAVE( CONSTEXPR_14 ) +# define optional_constexpr14 constexpr +#else +# define optional_constexpr14 /*constexpr*/ +#endif + +#if optional_HAVE( NODISCARD ) +# define optional_nodiscard [[nodiscard]] +#else +# define optional_nodiscard /*[[nodiscard]]*/ +#endif + +#if optional_HAVE( NOEXCEPT ) +# define optional_noexcept noexcept +#else +# define optional_noexcept /*noexcept*/ +#endif + +#if optional_HAVE( NULLPTR ) +# define optional_nullptr nullptr +#else +# define optional_nullptr NULL +#endif + +#if optional_HAVE( REF_QUALIFIER ) +// NOLINTNEXTLINE( bugprone-macro-parentheses ) +# define optional_ref_qual & +# define optional_refref_qual && +#else +# define optional_ref_qual /*&*/ +# define optional_refref_qual /*&&*/ +#endif + +// additional includes: + +#if optional_CONFIG_NO_EXCEPTIONS +// already included: +#else +# include +#endif + +#if optional_CPP11_OR_GREATER +# include +#endif + +#if optional_HAVE( INITIALIZER_LIST ) +# include +#endif + +#if optional_HAVE( TYPE_TRAITS ) +# include +#elif optional_HAVE( TR1_TYPE_TRAITS ) +# include +#endif + +// Method enabling + +#if optional_CPP11_OR_GREATER + +#define optional_REQUIRES_0(...) \ + template< bool B = (__VA_ARGS__), typename std::enable_if::type = 0 > + +#define optional_REQUIRES_T(...) \ + , typename std::enable_if< (__VA_ARGS__), int >::type = 0 + +#define optional_REQUIRES_R(R, ...) \ + typename std::enable_if< (__VA_ARGS__), R>::type + +#define optional_REQUIRES_A(...) \ + , typename std::enable_if< (__VA_ARGS__), void*>::type = nullptr + +#endif + +// +// optional: +// + +namespace nonstd { namespace optional_lite { + +namespace std11 { + +#if optional_CPP11_OR_GREATER + using std::move; +#else + template< typename T > T & move( T & t ) { return t; } +#endif + +#if optional_HAVE( CONDITIONAL ) + using std::conditional; +#else + template< bool B, typename T, typename F > struct conditional { typedef T type; }; + template< typename T, typename F > struct conditional { typedef F type; }; +#endif // optional_HAVE_CONDITIONAL + +// gcc < 5: +#if optional_CPP11_OR_GREATER +#if optional_BETWEEN( optional_COMPILER_GNUC_VERSION, 1, 500 ) + template< typename T > struct is_trivially_copy_constructible : std::true_type{}; + template< typename T > struct is_trivially_move_constructible : std::true_type{}; +#else + using std::is_trivially_copy_constructible; + using std::is_trivially_move_constructible; +#endif +#endif +} // namespace std11 + +#if optional_CPP11_OR_GREATER + +/// type traits C++17: + +namespace std17 { + +#if optional_CPP17_OR_GREATER + +using std::is_swappable; +using std::is_nothrow_swappable; + +#elif optional_CPP11_OR_GREATER + +namespace detail { + +using std::swap; + +struct is_swappable +{ + template< typename T, typename = decltype( swap( std::declval(), std::declval() ) ) > + static std::true_type test( int /*unused*/ ); + + template< typename > + static std::false_type test(...); +}; + +struct is_nothrow_swappable +{ + // wrap noexcept(expr) in separate function as work-around for VC140 (VS2015): + + template< typename T > + static constexpr bool satisfies() + { + return noexcept( swap( std::declval(), std::declval() ) ); + } + + template< typename T > + static auto test( int /*unused*/ ) -> std::integral_constant()>{} + + template< typename > + static auto test(...) -> std::false_type; +}; + +} // namespace detail + +// is [nothow] swappable: + +template< typename T > +struct is_swappable : decltype( detail::is_swappable::test(0) ){}; + +template< typename T > +struct is_nothrow_swappable : decltype( detail::is_nothrow_swappable::test(0) ){}; + +#endif // optional_CPP17_OR_GREATER + +} // namespace std17 + +/// type traits C++20: + +namespace std20 { + +template< typename T > +struct remove_cvref +{ + typedef typename std::remove_cv< typename std::remove_reference::type >::type type; +}; + +} // namespace std20 + +#endif // optional_CPP11_OR_GREATER + +/// class optional + +template< typename T > +class optional; + +namespace detail { + +// C++11 emulation: + +struct nulltype{}; + +template< typename Head, typename Tail > +struct typelist +{ + typedef Head head; + typedef Tail tail; +}; + +#if optional_CONFIG_MAX_ALIGN_HACK + +// Max align, use most restricted type for alignment: + +#define optional_UNIQUE( name ) optional_UNIQUE2( name, __LINE__ ) +#define optional_UNIQUE2( name, line ) optional_UNIQUE3( name, line ) +#define optional_UNIQUE3( name, line ) name ## line + +#define optional_ALIGN_TYPE( type ) \ + type optional_UNIQUE( _t ); struct_t< type > optional_UNIQUE( _st ) + +template< typename T > +struct struct_t { T _; }; + +union max_align_t +{ + optional_ALIGN_TYPE( char ); + optional_ALIGN_TYPE( short int ); + optional_ALIGN_TYPE( int ); + optional_ALIGN_TYPE( long int ); + optional_ALIGN_TYPE( float ); + optional_ALIGN_TYPE( double ); + optional_ALIGN_TYPE( long double ); + optional_ALIGN_TYPE( char * ); + optional_ALIGN_TYPE( short int * ); + optional_ALIGN_TYPE( int * ); + optional_ALIGN_TYPE( long int * ); + optional_ALIGN_TYPE( float * ); + optional_ALIGN_TYPE( double * ); + optional_ALIGN_TYPE( long double * ); + optional_ALIGN_TYPE( void * ); + +#ifdef HAVE_LONG_LONG + optional_ALIGN_TYPE( long long ); +#endif + + struct Unknown; + + Unknown ( * optional_UNIQUE(_) )( Unknown ); + Unknown * Unknown::* optional_UNIQUE(_); + Unknown ( Unknown::* optional_UNIQUE(_) )( Unknown ); + + struct_t< Unknown ( * )( Unknown) > optional_UNIQUE(_); + struct_t< Unknown * Unknown::* > optional_UNIQUE(_); + struct_t< Unknown ( Unknown::* )(Unknown) > optional_UNIQUE(_); +}; + +#undef optional_UNIQUE +#undef optional_UNIQUE2 +#undef optional_UNIQUE3 + +#undef optional_ALIGN_TYPE + +#elif defined( optional_CONFIG_ALIGN_AS ) // optional_CONFIG_MAX_ALIGN_HACK + +// Use user-specified type for alignment: + +#define optional_ALIGN_AS( unused ) \ + optional_CONFIG_ALIGN_AS + +#else // optional_CONFIG_MAX_ALIGN_HACK + +// Determine POD type to use for alignment: + +#define optional_ALIGN_AS( to_align ) \ + typename type_of_size< alignment_types, alignment_of< to_align >::value >::type + +template< typename T > +struct alignment_of; + +template< typename T > +struct alignment_of_hack +{ + char c; + T t; + alignment_of_hack(); +}; + +template< size_t A, size_t S > +struct alignment_logic +{ + enum { value = A < S ? A : S }; +}; + +template< typename T > +struct alignment_of +{ + enum { value = alignment_logic< + sizeof( alignment_of_hack ) - sizeof(T), sizeof(T) >::value }; +}; + +template< typename List, size_t N > +struct type_of_size +{ + typedef typename std11::conditional< + N == sizeof( typename List::head ), + typename List::head, + typename type_of_size::type >::type type; +}; + +template< size_t N > +struct type_of_size< nulltype, N > +{ + typedef optional_CONFIG_ALIGN_AS_FALLBACK type; +}; + +template< typename T> +struct struct_t { T _; }; + +#define optional_ALIGN_TYPE( type ) \ + typelist< type , typelist< struct_t< type > + +struct Unknown; + +typedef + optional_ALIGN_TYPE( char ), + optional_ALIGN_TYPE( short ), + optional_ALIGN_TYPE( int ), + optional_ALIGN_TYPE( long ), + optional_ALIGN_TYPE( float ), + optional_ALIGN_TYPE( double ), + optional_ALIGN_TYPE( long double ), + + optional_ALIGN_TYPE( char *), + optional_ALIGN_TYPE( short * ), + optional_ALIGN_TYPE( int * ), + optional_ALIGN_TYPE( long * ), + optional_ALIGN_TYPE( float * ), + optional_ALIGN_TYPE( double * ), + optional_ALIGN_TYPE( long double * ), + + optional_ALIGN_TYPE( Unknown ( * )( Unknown ) ), + optional_ALIGN_TYPE( Unknown * Unknown::* ), + optional_ALIGN_TYPE( Unknown ( Unknown::* )( Unknown ) ), + + nulltype + > > > > > > > > > > > > > > + > > > > > > > > > > > > > > + > > > > > > + alignment_types; + +#undef optional_ALIGN_TYPE + +#endif // optional_CONFIG_MAX_ALIGN_HACK + +/// C++03 constructed union to hold value. + +template< typename T > +union storage_t +{ +//private: +// template< typename > friend class optional; + + typedef T value_type; + + storage_t() optional_is_default + + explicit storage_t( value_type const & v ) + { + construct_value( v ); + } + + void construct_value( value_type const & v ) + { + ::new( value_ptr() ) value_type( v ); + } + +#if optional_CPP11_OR_GREATER + + explicit storage_t( value_type && v ) + { + construct_value( std::move( v ) ); + } + + void construct_value( value_type && v ) + { + ::new( value_ptr() ) value_type( std::move( v ) ); + } + + template< class... Args > + void emplace( Args&&... args ) + { + ::new( value_ptr() ) value_type( std::forward(args)... ); + } + + template< class U, class... Args > + void emplace( std::initializer_list il, Args&&... args ) + { + ::new( value_ptr() ) value_type( il, std::forward(args)... ); + } + +#endif + + void destruct_value() + { + value_ptr()->~T(); + } + + optional_nodiscard value_type const * value_ptr() const + { + return as(); + } + + value_type * value_ptr() + { + return as(); + } + + optional_nodiscard value_type const & value() const optional_ref_qual + { + return * value_ptr(); + } + + value_type & value() optional_ref_qual + { + return * value_ptr(); + } + +#if optional_HAVE( REF_QUALIFIER ) + + optional_nodiscard value_type const && value() const optional_refref_qual + { + return std::move( value() ); + } + + value_type && value() optional_refref_qual + { + return std::move( value() ); + } + +#endif + +#if optional_CPP11_OR_GREATER + + using aligned_storage_t = typename std::aligned_storage< sizeof(value_type), alignof(value_type) >::type; + aligned_storage_t data; + +#elif optional_CONFIG_MAX_ALIGN_HACK + + typedef struct { unsigned char data[ sizeof(value_type) ]; } aligned_storage_t; + + max_align_t hack; + aligned_storage_t data; + +#else + typedef optional_ALIGN_AS(value_type) align_as_type; + + typedef struct { align_as_type data[ 1 + ( sizeof(value_type) - 1 ) / sizeof(align_as_type) ]; } aligned_storage_t; + aligned_storage_t data; + +# undef optional_ALIGN_AS + +#endif // optional_CONFIG_MAX_ALIGN_HACK + + optional_nodiscard void * ptr() optional_noexcept + { + return &data; + } + + optional_nodiscard void const * ptr() const optional_noexcept + { + return &data; + } + + template + optional_nodiscard U * as() + { + return reinterpret_cast( ptr() ); + } + + template + optional_nodiscard U const * as() const + { + return reinterpret_cast( ptr() ); + } +}; + +} // namespace detail + +/// disengaged state tag + +struct nullopt_t +{ + struct init{}; + explicit optional_constexpr nullopt_t( init /*unused*/ ) optional_noexcept {} +}; + +#if optional_HAVE( CONSTEXPR_11 ) +constexpr nullopt_t nullopt{ nullopt_t::init{} }; +#else +// extra parenthesis to prevent the most vexing parse: +const nullopt_t nullopt(( nullopt_t::init() )); +#endif + +/// optional access error + +#if ! optional_CONFIG_NO_EXCEPTIONS + +class bad_optional_access : public std::logic_error +{ +public: + explicit bad_optional_access() + : logic_error( "bad optional access" ) {} +}; + +#endif //optional_CONFIG_NO_EXCEPTIONS + +/// optional + +template< typename T> +class optional +{ +private: + template< typename > friend class optional; + + typedef void (optional::*safe_bool)() const; + +public: + typedef T value_type; + + // x.x.3.1, constructors + + // 1a - default construct + optional_constexpr optional() optional_noexcept + : has_value_( false ) + , contained() + {} + + // 1b - construct explicitly empty + // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) + optional_constexpr optional( nullopt_t /*unused*/ ) optional_noexcept + : has_value_( false ) + , contained() + {} + + // 2 - copy-construct +#if optional_CPP11_OR_GREATER + // template< typename U = T + // optional_REQUIRES_T( + // std::is_copy_constructible::value + // || std11::is_trivially_copy_constructible::value + // ) + // > +#endif + optional_constexpr14 optional( optional const & other ) + : has_value_( other.has_value() ) + { + if ( other.has_value() ) + { + contained.construct_value( other.contained.value() ); + } + } + +#if optional_CPP11_OR_GREATER + + // 3 (C++11) - move-construct from optional + template< typename U = T + optional_REQUIRES_T( + std::is_move_constructible::value + || std11::is_trivially_move_constructible::value + ) + > + optional_constexpr14 optional( optional && other ) + // NOLINTNEXTLINE( performance-noexcept-move-constructor ) + noexcept( std::is_nothrow_move_constructible::value ) + : has_value_( other.has_value() ) + { + if ( other.has_value() ) + { + contained.construct_value( std::move( other.contained.value() ) ); + } + } + + // 4a (C++11) - explicit converting copy-construct from optional + template< typename U + optional_REQUIRES_T( + std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< optional & , T>::value + && !std::is_convertible< optional && , T>::value + && !std::is_convertible< optional const & , T>::value + && !std::is_convertible< optional const &&, T>::value + && !std::is_convertible< U const & , T>::value /*=> explicit */ + ) + > + explicit optional( optional const & other ) + : has_value_( other.has_value() ) + { + if ( other.has_value() ) + { + contained.construct_value( T{ other.contained.value() } ); + } + } +#endif // optional_CPP11_OR_GREATER + + // 4b (C++98 and later) - non-explicit converting copy-construct from optional + template< typename U +#if optional_CPP11_OR_GREATER + optional_REQUIRES_T( + std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< optional & , T>::value + && !std::is_convertible< optional && , T>::value + && !std::is_convertible< optional const & , T>::value + && !std::is_convertible< optional const &&, T>::value + && std::is_convertible< U const & , T>::value /*=> non-explicit */ + ) +#endif // optional_CPP11_OR_GREATER + > + // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) + /*non-explicit*/ optional( optional const & other ) + : has_value_( other.has_value() ) + { + if ( other.has_value() ) + { + contained.construct_value( other.contained.value() ); + } + } + +#if optional_CPP11_OR_GREATER + + // 5a (C++11) - explicit converting move-construct from optional + template< typename U + optional_REQUIRES_T( + std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< optional & , T>::value + && !std::is_convertible< optional && , T>::value + && !std::is_convertible< optional const & , T>::value + && !std::is_convertible< optional const &&, T>::value + && !std::is_convertible< U &&, T>::value /*=> explicit */ + ) + > + explicit optional( optional && other + ) + : has_value_( other.has_value() ) + { + if ( other.has_value() ) + { + contained.construct_value( T{ std::move( other.contained.value() ) } ); + } + } + + // 5a (C++11) - non-explicit converting move-construct from optional + template< typename U + optional_REQUIRES_T( + std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< optional & , T>::value + && !std::is_convertible< optional && , T>::value + && !std::is_convertible< optional const & , T>::value + && !std::is_convertible< optional const &&, T>::value + && std::is_convertible< U &&, T>::value /*=> non-explicit */ + ) + > + // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) + /*non-explicit*/ optional( optional && other ) + : has_value_( other.has_value() ) + { + if ( other.has_value() ) + { + contained.construct_value( std::move( other.contained.value() ) ); + } + } + + // 6 (C++11) - in-place construct + template< typename... Args + optional_REQUIRES_T( + std::is_constructible::value + ) + > + optional_constexpr explicit optional( nonstd_lite_in_place_t(T), Args&&... args ) + : has_value_( true ) + , contained( T( std::forward(args)...) ) + {} + + // 7 (C++11) - in-place construct, initializer-list + template< typename U, typename... Args + optional_REQUIRES_T( + std::is_constructible&, Args&&...>::value + ) + > + optional_constexpr explicit optional( nonstd_lite_in_place_t(T), std::initializer_list il, Args&&... args ) + : has_value_( true ) + , contained( T( il, std::forward(args)...) ) + {} + + // 8a (C++11) - explicit move construct from value + template< typename U = T + optional_REQUIRES_T( + std::is_constructible::value + && !std::is_same::type, nonstd_lite_in_place_t(U)>::value + && !std::is_same::type, optional>::value + && !std::is_convertible::value /*=> explicit */ + ) + > + optional_constexpr explicit optional( U && value ) + : has_value_( true ) + , contained( T{ std::forward( value ) } ) + {} + + // 8b (C++11) - non-explicit move construct from value + template< typename U = T + optional_REQUIRES_T( + std::is_constructible::value + && !std::is_same::type, nonstd_lite_in_place_t(U)>::value + && !std::is_same::type, optional>::value + && std::is_convertible::value /*=> non-explicit */ + ) + > + // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) + optional_constexpr /*non-explicit*/ optional( U && value ) + : has_value_( true ) + , contained( std::forward( value ) ) + {} + +#else // optional_CPP11_OR_GREATER + + // 8 (C++98) + optional( value_type const & value ) + : has_value_( true ) + , contained( value ) + {} + +#endif // optional_CPP11_OR_GREATER + + // x.x.3.2, destructor + + ~optional() + { + if ( has_value() ) + { + contained.destruct_value(); + } + } + + // x.x.3.3, assignment + + // 1 (C++98and later) - assign explicitly empty + optional & operator=( nullopt_t /*unused*/) optional_noexcept + { + reset(); + return *this; + } + + // 2 (C++98and later) - copy-assign from optional +#if optional_CPP11_OR_GREATER + // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator ) + optional_REQUIRES_R( + optional &, + true +// std::is_copy_constructible::value +// && std::is_copy_assignable::value + ) + operator=( optional const & other ) + noexcept( + std::is_nothrow_move_assignable::value + && std::is_nothrow_move_constructible::value + ) +#else + optional & operator=( optional const & other ) +#endif + { + if ( (has_value() == true ) && (other.has_value() == false) ) { reset(); } + else if ( (has_value() == false) && (other.has_value() == true ) ) { initialize( *other ); } + else if ( (has_value() == true ) && (other.has_value() == true ) ) { contained.value() = *other; } + return *this; + } + +#if optional_CPP11_OR_GREATER + + // 3 (C++11) - move-assign from optional + // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator ) + optional_REQUIRES_R( + optional &, + true +// std::is_move_constructible::value +// && std::is_move_assignable::value + ) + operator=( optional && other ) noexcept + { + if ( (has_value() == true ) && (other.has_value() == false) ) { reset(); } + else if ( (has_value() == false) && (other.has_value() == true ) ) { initialize( std::move( *other ) ); } + else if ( (has_value() == true ) && (other.has_value() == true ) ) { contained.value() = std::move( *other ); } + return *this; + } + + // 4 (C++11) - move-assign from value + template< typename U = T > + // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator ) + optional_REQUIRES_R( + optional &, + std::is_constructible::value + && std::is_assignable::value + && !std::is_same::type, nonstd_lite_in_place_t(U)>::value + && !std::is_same::type, optional>::value + && !(std::is_scalar::value && std::is_same::type>::value) + ) + operator=( U && value ) + { + if ( has_value() ) + { + contained.value() = std::forward( value ); + } + else + { + initialize( T( std::forward( value ) ) ); + } + return *this; + } + +#else // optional_CPP11_OR_GREATER + + // 4 (C++98) - copy-assign from value + template< typename U /*= T*/ > + optional & operator=( U const & value ) + { + if ( has_value() ) contained.value() = value; + else initialize( T( value ) ); + return *this; + } + +#endif // optional_CPP11_OR_GREATER + + // 5 (C++98 and later) - converting copy-assign from optional + template< typename U > +#if optional_CPP11_OR_GREATER + // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator ) + optional_REQUIRES_R( + optional&, + std::is_constructible< T , U const &>::value + && std::is_assignable< T&, U const &>::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< optional & , T>::value + && !std::is_convertible< optional && , T>::value + && !std::is_convertible< optional const & , T>::value + && !std::is_convertible< optional const &&, T>::value + && !std::is_assignable< T&, optional & >::value + && !std::is_assignable< T&, optional && >::value + && !std::is_assignable< T&, optional const & >::value + && !std::is_assignable< T&, optional const && >::value + ) +#else + optional& +#endif // optional_CPP11_OR_GREATER + operator=( optional const & other ) + { + return *this = optional( other ); + } + +#if optional_CPP11_OR_GREATER + + // 6 (C++11) - converting move-assign from optional + template< typename U > + // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator ) + optional_REQUIRES_R( + optional&, + std::is_constructible< T , U>::value + && std::is_assignable< T&, U>::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< optional & , T>::value + && !std::is_convertible< optional && , T>::value + && !std::is_convertible< optional const & , T>::value + && !std::is_convertible< optional const &&, T>::value + && !std::is_assignable< T&, optional & >::value + && !std::is_assignable< T&, optional && >::value + && !std::is_assignable< T&, optional const & >::value + && !std::is_assignable< T&, optional const && >::value + ) + operator=( optional && other ) + { + return *this = optional( std::move( other ) ); + } + + // 7 (C++11) - emplace + template< typename... Args + optional_REQUIRES_T( + std::is_constructible::value + ) + > + T& emplace( Args&&... args ) + { + *this = nullopt; + contained.emplace( std::forward(args)... ); + has_value_ = true; + return contained.value(); + } + + // 8 (C++11) - emplace, initializer-list + template< typename U, typename... Args + optional_REQUIRES_T( + std::is_constructible&, Args&&...>::value + ) + > + T& emplace( std::initializer_list il, Args&&... args ) + { + *this = nullopt; + contained.emplace( il, std::forward(args)... ); + has_value_ = true; + return contained.value(); + } + +#endif // optional_CPP11_OR_GREATER + + // x.x.3.4, swap + + void swap( optional & other ) +#if optional_CPP11_OR_GREATER + noexcept( + std::is_nothrow_move_constructible::value + && std17::is_nothrow_swappable::value + ) +#endif + { + using std::swap; + if ( (has_value() == true ) && (other.has_value() == true ) ) { swap( **this, *other ); } + else if ( (has_value() == false) && (other.has_value() == true ) ) { initialize( std11::move(*other) ); other.reset(); } + else if ( (has_value() == true ) && (other.has_value() == false) ) { other.initialize( std11::move(**this) ); reset(); } + } + + // x.x.3.5, observers + + optional_constexpr value_type const * operator ->() const + { + return assert( has_value() ), + contained.value_ptr(); + } + + optional_constexpr14 value_type * operator ->() + { + return assert( has_value() ), + contained.value_ptr(); + } + + optional_constexpr value_type const & operator *() const optional_ref_qual + { + return assert( has_value() ), + contained.value(); + } + + optional_constexpr14 value_type & operator *() optional_ref_qual + { + return assert( has_value() ), + contained.value(); + } + +#if optional_HAVE( REF_QUALIFIER ) + + optional_constexpr value_type const && operator *() const optional_refref_qual + { + return std::move( **this ); + } + + optional_constexpr14 value_type && operator *() optional_refref_qual + { + return std::move( **this ); + } + +#endif + +#if optional_CPP11_OR_GREATER + optional_constexpr explicit operator bool() const optional_noexcept + { + return has_value(); + } +#else + optional_constexpr operator safe_bool() const optional_noexcept + { + return has_value() ? &optional::this_type_does_not_support_comparisons : 0; + } +#endif + + // NOLINTNEXTLINE( modernize-use-nodiscard ) + /*optional_nodiscard*/ optional_constexpr bool has_value() const optional_noexcept + { + return has_value_; + } + + // NOLINTNEXTLINE( modernize-use-nodiscard ) + /*optional_nodiscard*/ optional_constexpr14 value_type const & value() const optional_ref_qual + { +#if optional_CONFIG_NO_EXCEPTIONS + assert( has_value() ); +#else + if ( ! has_value() ) + { + throw bad_optional_access(); + } +#endif + return contained.value(); + } + + optional_constexpr14 value_type & value() optional_ref_qual + { +#if optional_CONFIG_NO_EXCEPTIONS + assert( has_value() ); +#else + if ( ! has_value() ) + { + throw bad_optional_access(); + } +#endif + return contained.value(); + } + +#if optional_HAVE( REF_QUALIFIER ) && ( !optional_COMPILER_GNUC_VERSION || optional_COMPILER_GNUC_VERSION >= 490 ) + + // NOLINTNEXTLINE( modernize-use-nodiscard ) + /*optional_nodiscard*/ optional_constexpr value_type const && value() const optional_refref_qual + { + return std::move( value() ); + } + + optional_constexpr14 value_type && value() optional_refref_qual + { + return std::move( value() ); + } + +#endif + +#if optional_CPP11_OR_GREATER + + template< typename U > + optional_constexpr value_type value_or( U && v ) const optional_ref_qual + { + return has_value() ? contained.value() : static_cast(std::forward( v ) ); + } + + template< typename U > + optional_constexpr14 value_type value_or( U && v ) optional_refref_qual + { + return has_value() ? std::move( contained.value() ) : static_cast(std::forward( v ) ); + } + +#else + + template< typename U > + optional_constexpr value_type value_or( U const & v ) const + { + return has_value() ? contained.value() : static_cast( v ); + } + +#endif // optional_CPP11_OR_GREATER + + // x.x.3.6, modifiers + + void reset() optional_noexcept + { + if ( has_value() ) + { + contained.destruct_value(); + } + + has_value_ = false; + } + +private: + void this_type_does_not_support_comparisons() const {} + + template< typename V > + void initialize( V const & value ) + { + assert( ! has_value() ); + contained.construct_value( value ); + has_value_ = true; + } + +#if optional_CPP11_OR_GREATER + template< typename V > + void initialize( V && value ) + { + assert( ! has_value() ); + contained.construct_value( std::move( value ) ); + has_value_ = true; + } + +#endif + +private: + bool has_value_; + detail::storage_t< value_type > contained; + +}; + +// Relational operators + +template< typename T, typename U > +inline optional_constexpr bool operator==( optional const & x, optional const & y ) +{ + return bool(x) != bool(y) ? false : !bool( x ) ? true : *x == *y; +} + +template< typename T, typename U > +inline optional_constexpr bool operator!=( optional const & x, optional const & y ) +{ + return !(x == y); +} + +template< typename T, typename U > +inline optional_constexpr bool operator<( optional const & x, optional const & y ) +{ + return (!y) ? false : (!x) ? true : *x < *y; +} + +template< typename T, typename U > +inline optional_constexpr bool operator>( optional const & x, optional const & y ) +{ + return (y < x); +} + +template< typename T, typename U > +inline optional_constexpr bool operator<=( optional const & x, optional const & y ) +{ + return !(y < x); +} + +template< typename T, typename U > +inline optional_constexpr bool operator>=( optional const & x, optional const & y ) +{ + return !(x < y); +} + +// Comparison with nullopt + +template< typename T > +inline optional_constexpr bool operator==( optional const & x, nullopt_t /*unused*/ ) optional_noexcept +{ + return (!x); +} + +template< typename T > +inline optional_constexpr bool operator==( nullopt_t /*unused*/, optional const & x ) optional_noexcept +{ + return (!x); +} + +template< typename T > +inline optional_constexpr bool operator!=( optional const & x, nullopt_t /*unused*/ ) optional_noexcept +{ + return bool(x); +} + +template< typename T > +inline optional_constexpr bool operator!=( nullopt_t /*unused*/, optional const & x ) optional_noexcept +{ + return bool(x); +} + +template< typename T > +inline optional_constexpr bool operator<( optional const & /*unused*/, nullopt_t /*unused*/ ) optional_noexcept +{ + return false; +} + +template< typename T > +inline optional_constexpr bool operator<( nullopt_t /*unused*/, optional const & x ) optional_noexcept +{ + return bool(x); +} + +template< typename T > +inline optional_constexpr bool operator<=( optional const & x, nullopt_t /*unused*/ ) optional_noexcept +{ + return (!x); +} + +template< typename T > +inline optional_constexpr bool operator<=( nullopt_t /*unused*/, optional const & /*unused*/ ) optional_noexcept +{ + return true; +} + +template< typename T > +inline optional_constexpr bool operator>( optional const & x, nullopt_t /*unused*/ ) optional_noexcept +{ + return bool(x); +} + +template< typename T > +inline optional_constexpr bool operator>( nullopt_t /*unused*/, optional const & /*unused*/ ) optional_noexcept +{ + return false; +} + +template< typename T > +inline optional_constexpr bool operator>=( optional const & /*unused*/, nullopt_t /*unused*/ ) optional_noexcept +{ + return true; +} + +template< typename T > +inline optional_constexpr bool operator>=( nullopt_t /*unused*/, optional const & x ) optional_noexcept +{ + return (!x); +} + +// Comparison with T + +template< typename T, typename U > +inline optional_constexpr bool operator==( optional const & x, U const & v ) +{ + return bool(x) ? *x == v : false; +} + +template< typename T, typename U > +inline optional_constexpr bool operator==( U const & v, optional const & x ) +{ + return bool(x) ? v == *x : false; +} + +template< typename T, typename U > +inline optional_constexpr bool operator!=( optional const & x, U const & v ) +{ + return bool(x) ? *x != v : true; +} + +template< typename T, typename U > +inline optional_constexpr bool operator!=( U const & v, optional const & x ) +{ + return bool(x) ? v != *x : true; +} + +template< typename T, typename U > +inline optional_constexpr bool operator<( optional const & x, U const & v ) +{ + return bool(x) ? *x < v : true; +} + +template< typename T, typename U > +inline optional_constexpr bool operator<( U const & v, optional const & x ) +{ + return bool(x) ? v < *x : false; +} + +template< typename T, typename U > +inline optional_constexpr bool operator<=( optional const & x, U const & v ) +{ + return bool(x) ? *x <= v : true; +} + +template< typename T, typename U > +inline optional_constexpr bool operator<=( U const & v, optional const & x ) +{ + return bool(x) ? v <= *x : false; +} + +template< typename T, typename U > +inline optional_constexpr bool operator>( optional const & x, U const & v ) +{ + return bool(x) ? *x > v : false; +} + +template< typename T, typename U > +inline optional_constexpr bool operator>( U const & v, optional const & x ) +{ + return bool(x) ? v > *x : true; +} + +template< typename T, typename U > +inline optional_constexpr bool operator>=( optional const & x, U const & v ) +{ + return bool(x) ? *x >= v : false; +} + +template< typename T, typename U > +inline optional_constexpr bool operator>=( U const & v, optional const & x ) +{ + return bool(x) ? v >= *x : true; +} + +// Specialized algorithms + +template< typename T +#if optional_CPP11_OR_GREATER + optional_REQUIRES_T( + std::is_move_constructible::value + && std17::is_swappable::value ) +#endif +> +void swap( optional & x, optional & y ) +#if optional_CPP11_OR_GREATER + noexcept( noexcept( x.swap(y) ) ) +#endif +{ + x.swap( y ); +} + +#if optional_CPP11_OR_GREATER + +template< typename T > +optional_constexpr optional< typename std::decay::type > make_optional( T && value ) +{ + return optional< typename std::decay::type >( std::forward( value ) ); +} + +template< typename T, typename...Args > +optional_constexpr optional make_optional( Args&&... args ) +{ + return optional( nonstd_lite_in_place(T), std::forward(args)...); +} + +template< typename T, typename U, typename... Args > +optional_constexpr optional make_optional( std::initializer_list il, Args&&... args ) +{ + return optional( nonstd_lite_in_place(T), il, std::forward(args)...); +} + +#else + +template< typename T > +optional make_optional( T const & value ) +{ + return optional( value ); +} + +#endif // optional_CPP11_OR_GREATER + +} // namespace optional_lite + +using optional_lite::optional; +using optional_lite::nullopt_t; +using optional_lite::nullopt; + +#if ! optional_CONFIG_NO_EXCEPTIONS +using optional_lite::bad_optional_access; +#endif + +using optional_lite::make_optional; + +} // namespace nonstd + +#if optional_CPP11_OR_GREATER + +// specialize the std::hash algorithm: + +namespace std { + +template< class T > +struct hash< nonstd::optional > +{ +public: + std::size_t operator()( nonstd::optional const & v ) const optional_noexcept + { + return bool( v ) ? std::hash{}( *v ) : 0; + } +}; + +} //namespace std + +#endif // optional_CPP11_OR_GREATER + +#if defined(__clang__) +# pragma clang diagnostic pop +#elif defined(__GNUC__) +# pragma GCC diagnostic pop +#elif defined(_MSC_VER ) +# pragma warning( pop ) +#endif + +#endif // optional_USES_STD_OPTIONAL + + +namespace tf { namespace nstd { + +template +using optional = nonstd::optional; + +}} diff --git a/bundled/cpp-taskflow/include/taskflow/nstd/variant.hpp b/bundled/cpp-taskflow/include/taskflow/nstd/variant.hpp new file mode 100644 index 0000000000..c07ee2057d --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/nstd/variant.hpp @@ -0,0 +1,2631 @@ +// The file is modified from MPark in order to use variant in C++11/14 +// We change the namespace to stick with the v1.4, in order to avoid clashing +// problem when upstream code uses the same library. + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt) + +// MSVC 2015 Update 3. +#if __cplusplus < 201103L && (!defined(_MSC_VER) || _MSC_FULL_VER < 190024210) +#error "MPark.Variant requires C++11 support." +#endif + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#ifndef __has_include +#define __has_include(x) 0 +#endif + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if __has_attribute(always_inline) || defined(__GNUC__) +#define TF_ALWAYS_INLINE __attribute__((__always_inline__)) inline +#elif defined(_MSC_VER) +#define TF_ALWAYS_INLINE __forceinline +#else +#define TF_ALWAYS_INLINE inline +#endif + +#if __has_builtin(__builtin_addressof) || \ + (defined(__GNUC__) && __GNUC__ >= 7) || defined(_MSC_VER) +#define TF_BUILTIN_ADDRESSOF +#endif + +#if __has_builtin(__builtin_unreachable) || defined(__GNUC__) +#define TF_BUILTIN_UNREACHABLE __builtin_unreachable() +#elif defined(_MSC_VER) +#define TF_BUILTIN_UNREACHABLE __assume(false) +#else +#define TF_BUILTIN_UNREACHABLE +#endif + +#if __has_builtin(__type_pack_element) +#define TF_TYPE_PACK_ELEMENT +#endif + +#if defined(__cpp_constexpr) && __cpp_constexpr >= 200704 && \ + !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 9) +#define TF_CPP11_CONSTEXPR +#endif + +#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304 +#define TF_CPP14_CONSTEXPR +#endif + +#if __has_feature(cxx_exceptions) || defined(__cpp_exceptions) || \ + (defined(_MSC_VER) && defined(_CPPUNWIND)) +#define TF_EXCEPTIONS +#endif + +#if defined(__cpp_generic_lambdas) || defined(_MSC_VER) +#define TF_GENERIC_LAMBDAS +#endif + +#if defined(__cpp_lib_integer_sequence) +#define TF_INTEGER_SEQUENCE +#endif + +#if (defined(__cpp_decltype_auto) && defined(__cpp_return_type_deduction)) || defined(_MSC_VER) +#define TF_RETURN_TYPE_DEDUCTION +#endif + +#if defined(__cpp_lib_transparent_operators) || defined(_MSC_VER) +#define TF_TRANSPARENT_OPERATORS +#endif + +#if defined(__cpp_variable_templates) || defined(_MSC_VER) +#define TF_VARIABLE_TEMPLATES +#endif + +#if !defined(__GLIBCXX__) || __has_include() // >= libstdc++-5 +#define TF_TRIVIALITY_TYPE_TRAITS +#define TF_INCOMPLETE_TYPE_TRAITS +#endif + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt) + +// in_place_t definition +namespace tf { namespace nstd { + + struct in_place_t { explicit in_place_t() = default; }; + + template + struct in_place_index_t { explicit in_place_index_t() = default; }; + + template + struct in_place_type_t { explicit in_place_type_t() = default; }; + +#ifdef TF_VARIABLE_TEMPLATES + constexpr in_place_t in_place{}; + + template constexpr in_place_index_t in_place_index{}; + + template constexpr in_place_type_t in_place_type{}; +#endif + +}} // namespace tf::nstd + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt) + + +#define TF_RETURN(...) \ + noexcept(noexcept(__VA_ARGS__)) -> decltype(__VA_ARGS__) { return __VA_ARGS__; } + +namespace tf { namespace nstd { + namespace lib { + template + struct identity { using type = T; }; + + inline namespace cpp14 { + template + struct array { + constexpr const T &operator[](std::size_t index) const { + return data[index]; + } + + T data[N == 0 ? 1 : N]; + }; + + template + using add_pointer_t = typename std::add_pointer::type; + + template + using common_type_t = typename std::common_type::type; + + template + using decay_t = typename std::decay::type; + + template + using enable_if_t = typename std::enable_if::type; + + template + using remove_const_t = typename std::remove_const::type; + + template + using remove_reference_t = typename std::remove_reference::type; + + template + using remove_cvref_t = + typename std::remove_cv>::type; + + template + inline constexpr T &&forward(remove_reference_t &t) noexcept { + return static_cast(t); + } + + template + inline constexpr T &&forward(remove_reference_t &&t) noexcept { + static_assert(!std::is_lvalue_reference::value, + "can not forward an rvalue as an lvalue"); + return static_cast(t); + } + + template + inline constexpr remove_reference_t &&move(T &&t) noexcept { + return static_cast &&>(t); + } + +#ifdef TF_INTEGER_SEQUENCE + using std::integer_sequence; + using std::index_sequence; + using std::make_index_sequence; + using std::index_sequence_for; +#else + template + struct integer_sequence { + using value_type = T; + static constexpr std::size_t size() noexcept { return sizeof...(Is); } + }; + + template + using index_sequence = integer_sequence; + + template + struct make_index_sequence_concat; + + template + struct make_index_sequence_concat, + index_sequence> + : identity> {}; + + template + struct make_index_sequence_impl; + + template + using make_index_sequence = typename make_index_sequence_impl::type; + + template + struct make_index_sequence_impl + : make_index_sequence_concat, + make_index_sequence> {}; + + template <> + struct make_index_sequence_impl<0> : identity> {}; + + template <> + struct make_index_sequence_impl<1> : identity> {}; + + template + using index_sequence_for = make_index_sequence; +#endif + + // +#ifdef TF_TRANSPARENT_OPERATORS + using equal_to = std::equal_to<>; +#else + struct equal_to { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + TF_RETURN(lib::forward(lhs) == lib::forward(rhs)) + }; +#endif + +#ifdef TF_TRANSPARENT_OPERATORS + using not_equal_to = std::not_equal_to<>; +#else + struct not_equal_to { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + TF_RETURN(lib::forward(lhs) != lib::forward(rhs)) + }; +#endif + +#ifdef TF_TRANSPARENT_OPERATORS + using less = std::less<>; +#else + struct less { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + TF_RETURN(lib::forward(lhs) < lib::forward(rhs)) + }; +#endif + +#ifdef TF_TRANSPARENT_OPERATORS + using greater = std::greater<>; +#else + struct greater { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + TF_RETURN(lib::forward(lhs) > lib::forward(rhs)) + }; +#endif + +#ifdef TF_TRANSPARENT_OPERATORS + using less_equal = std::less_equal<>; +#else + struct less_equal { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + TF_RETURN(lib::forward(lhs) <= lib::forward(rhs)) + }; +#endif + +#ifdef TF_TRANSPARENT_OPERATORS + using greater_equal = std::greater_equal<>; +#else + struct greater_equal { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + TF_RETURN(lib::forward(lhs) >= lib::forward(rhs)) + }; +#endif + } // namespace cpp14 + + inline namespace cpp17 { + + // + template + using bool_constant = std::integral_constant; + + template + struct voider : identity {}; + + template + using void_t = typename voider::type; + + namespace detail { + namespace swappable { + + using std::swap; + + template + struct is_swappable { + private: + template (), + std::declval()))> + inline static std::true_type test(int); + + template + inline static std::false_type test(...); + + public: + static constexpr bool value = decltype(test(0))::value; + }; + + template + struct is_nothrow_swappable { + static constexpr bool value = + noexcept(swap(std::declval(), std::declval())); + }; + + template + struct is_nothrow_swappable : std::false_type {}; + + } // namespace swappable + } // namespace detail + + using detail::swappable::is_swappable; + + template + using is_nothrow_swappable = + detail::swappable::is_nothrow_swappable::value, T>; + + // + namespace detail { + + template + struct is_reference_wrapper : std::false_type {}; + + template + struct is_reference_wrapper> + : std::true_type {}; + + template + struct Invoke; + + template <> + struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + TF_RETURN((lib::forward(arg).*pmf)(lib::forward(args)...)) + }; + + template <> + struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + TF_RETURN((lib::forward(arg).get().*pmf)(lib::forward(args)...)) + }; + + template <> + struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + TF_RETURN(((*lib::forward(arg)).*pmf)(lib::forward(args)...)) + }; + + template <> + struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + TF_RETURN(lib::forward(arg).*pmo) + }; + + template <> + struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + TF_RETURN(lib::forward(arg).get().*pmo) + }; + + template <> + struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + TF_RETURN((*lib::forward(arg)).*pmo) + }; + + template + inline constexpr auto invoke(R T::*f, Arg &&arg, Args &&... args) + TF_RETURN( + Invoke::value, + (std::is_base_of>::value + ? 0 + : is_reference_wrapper>::value + ? 1 + : 2)>::invoke(f, + lib::forward(arg), + lib::forward(args)...)) + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template + inline constexpr auto invoke(F &&f, Args &&... args) + TF_RETURN(lib::forward(f)(lib::forward(args)...)) +#ifdef _MSC_VER +#pragma warning(pop) +#endif + } // namespace detail + + template + inline constexpr auto invoke(F &&f, Args &&... args) + TF_RETURN(detail::invoke(lib::forward(f), + lib::forward(args)...)) + + namespace detail { + + template + struct invoke_result {}; + + template + struct invoke_result(), std::declval()...))>, + F, + Args...> + : identity(), std::declval()...))> {}; + + } // namespace detail + + template + using invoke_result = detail::invoke_result; + + template + using invoke_result_t = typename invoke_result::type; + + namespace detail { + + template + struct is_invocable : std::false_type {}; + + template + struct is_invocable>, F, Args...> + : std::true_type {}; + + template + struct is_invocable_r : std::false_type {}; + + template + struct is_invocable_r>, + R, + F, + Args...> + : std::is_convertible, R> {}; + + } // namespace detail + + template + using is_invocable = detail::is_invocable; + + template + using is_invocable_r = detail::is_invocable_r; + + // +#ifdef TF_BUILTIN_ADDRESSOF + template + inline constexpr T *addressof(T &arg) noexcept { + return __builtin_addressof(arg); + } +#else + namespace detail { + + namespace has_addressof_impl { + + struct fail; + + template + inline fail operator&(T &&); + + template + inline static constexpr bool impl() { + return (std::is_class::value || std::is_union::value) && + !std::is_same()), fail>::value; + } + + } // namespace has_addressof_impl + + template + using has_addressof = bool_constant()>; + + template + inline constexpr T *addressof(T &arg, std::true_type) noexcept { + return std::addressof(arg); + } + + template + inline constexpr T *addressof(T &arg, std::false_type) noexcept { + return &arg; + } + + } // namespace detail + + template + inline constexpr T *addressof(T &arg) noexcept { + return detail::addressof(arg, detail::has_addressof{}); + } +#endif + + template + inline constexpr T *addressof(const T &&) = delete; + + } // namespace cpp17 + + template + struct remove_all_extents : identity {}; + + template + struct remove_all_extents> : remove_all_extents {}; + + template + using remove_all_extents_t = typename remove_all_extents::type; + + template + using size_constant = std::integral_constant; + + template + struct indexed_type : size_constant { using type = T; }; + + template + using all = std::is_same, + integer_sequence>; + +#ifdef TF_TYPE_PACK_ELEMENT + template + using type_pack_element_t = __type_pack_element; +#else + template + struct type_pack_element_impl { + private: + template + struct set; + + template + struct set> : indexed_type... {}; + + template + inline static std::enable_if impl(indexed_type); + + inline static std::enable_if impl(...); + + public: + using type = decltype(impl(set>{})); + }; + + template + using type_pack_element = typename type_pack_element_impl::type; + + template + using type_pack_element_t = typename type_pack_element::type; +#endif + +#ifdef TF_TRIVIALITY_TYPE_TRAITS + using std::is_trivially_copy_constructible; + using std::is_trivially_move_constructible; + using std::is_trivially_copy_assignable; + using std::is_trivially_move_assignable; +#else + template + struct is_trivially_copy_constructible + : bool_constant< + std::is_copy_constructible::value && __has_trivial_copy(T)> {}; + + template + struct is_trivially_move_constructible : bool_constant<__is_trivial(T)> {}; + + template + struct is_trivially_copy_assignable + : bool_constant< + std::is_copy_assignable::value && __has_trivial_assign(T)> {}; + + template + struct is_trivially_move_assignable : bool_constant<__is_trivial(T)> {}; +#endif + + template + struct dependent_type : T {}; + + template + struct push_back; + + template + using push_back_t = typename push_back::type; + + template + struct push_back, J> { + using type = index_sequence; + }; + + } // namespace lib +}} // namespace tf::nstd + +#undef TF_RETURN + + +namespace tf { namespace nstd { + +#ifdef TF_RETURN_TYPE_DEDUCTION + +#define AUTO auto +#define AUTO_RETURN(...) { return __VA_ARGS__; } + +#define AUTO_REFREF auto && +#define AUTO_REFREF_RETURN(...) { return __VA_ARGS__; } + +#define DECLTYPE_AUTO decltype(auto) +#define DECLTYPE_AUTO_RETURN(...) { return __VA_ARGS__; } + +#else + +#define AUTO auto +#define AUTO_RETURN(...) \ + -> lib::decay_t { return __VA_ARGS__; } + +#define AUTO_REFREF auto +#define AUTO_REFREF_RETURN(...) \ + -> decltype((__VA_ARGS__)) { \ + static_assert(std::is_reference::value, ""); \ + return __VA_ARGS__; \ + } + +#define DECLTYPE_AUTO auto +#define DECLTYPE_AUTO_RETURN(...) \ + -> decltype(__VA_ARGS__) { return __VA_ARGS__; } + +#endif + + class bad_variant_access : public std::exception { + public: + virtual const char *what() const noexcept override { return "bad_variant_access"; } + }; + + [[noreturn]] inline void throw_bad_variant_access() { +#ifdef TF_EXCEPTIONS + throw bad_variant_access{}; +#else + std::terminate(); + TF_BUILTIN_UNREACHABLE; +#endif + } + + template + class variant; + + template + struct variant_size; + +#ifdef TF_VARIABLE_TEMPLATES + template + constexpr std::size_t variant_size_v = variant_size::value; +#endif + + template + struct variant_size : variant_size {}; + + template + struct variant_size : variant_size {}; + + template + struct variant_size : variant_size {}; + + template + struct variant_size> : lib::size_constant {}; + + template + struct variant_alternative; + + template + using variant_alternative_t = typename variant_alternative::type; + + template + struct variant_alternative + : std::add_const> {}; + + template + struct variant_alternative + : std::add_volatile> {}; + + template + struct variant_alternative + : std::add_cv> {}; + + template + struct variant_alternative> { + static_assert(I < sizeof...(Ts), + "index out of bounds in `std::variant_alternative<>`"); + using type = lib::type_pack_element_t; + }; + + constexpr std::size_t variant_npos = static_cast(-1); + + namespace detail { + + constexpr std::size_t not_found = static_cast(-1); + constexpr std::size_t ambiguous = static_cast(-2); + +#ifdef TF_CPP14_CONSTEXPR + template + inline constexpr std::size_t find_index() { + constexpr lib::array matches = { + {std::is_same::value...} + }; + std::size_t result = not_found; + for (std::size_t i = 0; i < sizeof...(Ts); ++i) { + if (matches[i]) { + if (result != not_found) { + return ambiguous; + } + result = i; + } + } + return result; + } +#else + inline constexpr std::size_t find_index_impl(std::size_t result, + std::size_t) { + return result; + } + + template + inline constexpr std::size_t find_index_impl(std::size_t result, + std::size_t idx, + bool b, + Bs... bs) { + return b ? (result != not_found ? ambiguous + : find_index_impl(idx, idx + 1, bs...)) + : find_index_impl(result, idx + 1, bs...); + } + + template + inline constexpr std::size_t find_index() { + return find_index_impl(not_found, 0, std::is_same::value...); + } +#endif + + template + using find_index_sfinae_impl = + lib::enable_if_t>; + + template + using find_index_sfinae = find_index_sfinae_impl()>; + + template + struct find_index_checked_impl : lib::size_constant { + static_assert(I != not_found, "the specified type is not found."); + static_assert(I != ambiguous, "the specified type is ambiguous."); + }; + + template + using find_index_checked = find_index_checked_impl()>; + + struct valueless_t {}; + + enum class Trait { TriviallyAvailable, Available, Unavailable }; + + template class IsTriviallyAvailable, + template class IsAvailable> + inline constexpr Trait trait() { + return IsTriviallyAvailable::value + ? Trait::TriviallyAvailable + : IsAvailable::value ? Trait::Available + : Trait::Unavailable; + } + +#ifdef TF_CPP14_CONSTEXPR + template + inline constexpr Trait common_trait(Traits... traits_) { + Trait result = Trait::TriviallyAvailable; + lib::array traits = {{traits_...}}; + for (std::size_t i = 0; i < sizeof...(Traits); ++i) { + Trait t = traits[i]; + if (static_cast(t) > static_cast(result)) { + result = t; + } + } + return result; + } +#else + inline constexpr Trait common_trait_impl(Trait result) { return result; } + + template + inline constexpr Trait common_trait_impl(Trait result, + Trait t, + Traits... ts) { + return static_cast(t) > static_cast(result) + ? common_trait_impl(t, ts...) + : common_trait_impl(result, ts...); + } + + template + inline constexpr Trait common_trait(Traits... ts) { + return common_trait_impl(Trait::TriviallyAvailable, ts...); + } +#endif + + template + struct traits { + static constexpr Trait copy_constructible_trait = + common_trait(trait()...); + + static constexpr Trait move_constructible_trait = + common_trait(trait()...); + + static constexpr Trait copy_assignable_trait = + common_trait(copy_constructible_trait, + trait()...); + + static constexpr Trait move_assignable_trait = + common_trait(move_constructible_trait, + trait()...); + + static constexpr Trait destructible_trait = + common_trait(trait()...); + }; + + namespace access { + + struct recursive_union { +#ifdef TF_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto &&get_alt(V &&v, in_place_index_t<0>) { + return lib::forward(v).head_; + } + + template + inline static constexpr auto &&get_alt(V &&v, in_place_index_t) { + return get_alt(lib::forward(v).tail_, in_place_index_t{}); + } +#else + template + struct get_alt_impl { + template + inline constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(get_alt_impl{}(lib::forward(v).tail_)) + }; + + template + struct get_alt_impl<0, Dummy> { + template + inline constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(lib::forward(v).head_) + }; + + template + inline static constexpr AUTO_REFREF get_alt(V &&v, in_place_index_t) + AUTO_REFREF_RETURN(get_alt_impl{}(lib::forward(v))) +#endif + }; + + struct base { + template + inline static constexpr AUTO_REFREF get_alt(V &&v) +#ifdef _MSC_VER + AUTO_REFREF_RETURN(recursive_union::get_alt( + lib::forward(v).data_, in_place_index_t{})) +#else + AUTO_REFREF_RETURN(recursive_union::get_alt( + data(lib::forward(v)), in_place_index_t{})) +#endif + }; + + struct variant { + template + inline static constexpr AUTO_REFREF get_alt(V &&v) + AUTO_REFREF_RETURN(base::get_alt(lib::forward(v).impl_)) + }; + + } // namespace access + + namespace visitation { + +#if defined(TF_CPP14_CONSTEXPR) && !defined(_MSC_VER) +#define TF_VARIANT_SWITCH_VISIT +#endif + + struct base { + template + using dispatch_result_t = decltype( + lib::invoke(std::declval(), + access::base::get_alt<0>(std::declval())...)); + + template + struct expected { + template + inline static constexpr bool but_got() { + return std::is_same::value; + } + }; + + template + struct visit_return_type_check { + static_assert( + expected::template but_got(), + "`visit` requires the visitor to have a single return type"); + + template + inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor, + Alts &&... alts) + DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward(visitor), + lib::forward(alts)...)) + }; + +#ifdef TF_VARIANT_SWITCH_VISIT + template + struct dispatcher; + + template + struct dispatcher { + template + TF_ALWAYS_INLINE static constexpr R dispatch( + F &&, typename ITs::type &&..., Vs &&...) { + TF_BUILTIN_UNREACHABLE; + } + + template + TF_ALWAYS_INLINE static constexpr R dispatch_case(F &&, Vs &&...) { + TF_BUILTIN_UNREACHABLE; + } + + template + TF_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t, + F &&, + Vs &&...) { + TF_BUILTIN_UNREACHABLE; + } + }; + + template + struct dispatcher { + template + TF_ALWAYS_INLINE static constexpr R dispatch( + F &&f, typename ITs::type &&... visited_vs) { + using Expected = R; + using Actual = decltype(lib::invoke( + lib::forward(f), + access::base::get_alt( + lib::forward(visited_vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt( + lib::forward(visited_vs))...); + } + + template + TF_ALWAYS_INLINE static constexpr R dispatch( + F &&f, typename ITs::type &&... visited_vs, V &&v, Vs &&... vs) { +#define TF_DISPATCH(I) \ + dispatcher<(I < lib::decay_t::size()), \ + R, \ + ITs..., \ + lib::indexed_type>:: \ + template dispatch<0>(lib::forward(f), \ + lib::forward(visited_vs)..., \ + lib::forward(v), \ + lib::forward(vs)...) + +#define TF_DEFAULT(I) \ + dispatcher<(I < lib::decay_t::size()), R, ITs...>::template dispatch( \ + lib::forward(f), \ + lib::forward(visited_vs)..., \ + lib::forward(v), \ + lib::forward(vs)...) + + switch (v.index()) { + case B + 0: return TF_DISPATCH(B + 0); + case B + 1: return TF_DISPATCH(B + 1); + case B + 2: return TF_DISPATCH(B + 2); + case B + 3: return TF_DISPATCH(B + 3); + case B + 4: return TF_DISPATCH(B + 4); + case B + 5: return TF_DISPATCH(B + 5); + case B + 6: return TF_DISPATCH(B + 6); + case B + 7: return TF_DISPATCH(B + 7); + case B + 8: return TF_DISPATCH(B + 8); + case B + 9: return TF_DISPATCH(B + 9); + case B + 10: return TF_DISPATCH(B + 10); + case B + 11: return TF_DISPATCH(B + 11); + case B + 12: return TF_DISPATCH(B + 12); + case B + 13: return TF_DISPATCH(B + 13); + case B + 14: return TF_DISPATCH(B + 14); + case B + 15: return TF_DISPATCH(B + 15); + case B + 16: return TF_DISPATCH(B + 16); + case B + 17: return TF_DISPATCH(B + 17); + case B + 18: return TF_DISPATCH(B + 18); + case B + 19: return TF_DISPATCH(B + 19); + case B + 20: return TF_DISPATCH(B + 20); + case B + 21: return TF_DISPATCH(B + 21); + case B + 22: return TF_DISPATCH(B + 22); + case B + 23: return TF_DISPATCH(B + 23); + case B + 24: return TF_DISPATCH(B + 24); + case B + 25: return TF_DISPATCH(B + 25); + case B + 26: return TF_DISPATCH(B + 26); + case B + 27: return TF_DISPATCH(B + 27); + case B + 28: return TF_DISPATCH(B + 28); + case B + 29: return TF_DISPATCH(B + 29); + case B + 30: return TF_DISPATCH(B + 30); + case B + 31: return TF_DISPATCH(B + 31); + default: return TF_DEFAULT(B + 32); + } + +#undef TF_DEFAULT +#undef TF_DISPATCH + } + + template + TF_ALWAYS_INLINE static constexpr R dispatch_case(F &&f, + Vs &&... vs) { + using Expected = R; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + + template + TF_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t index, + F &&f, + V &&v, + Vs &&... vs) { + static_assert(lib::all<(lib::decay_t::size() == + lib::decay_t::size())...>::value, + "all of the variants must be the same size."); +#define TF_DISPATCH_AT(I) \ + dispatcher<(I < lib::decay_t::size()), R>::template dispatch_case( \ + lib::forward(f), lib::forward(v), lib::forward(vs)...) + +#define TF_DEFAULT(I) \ + dispatcher<(I < lib::decay_t::size()), R>::template dispatch_at( \ + index, lib::forward(f), lib::forward(v), lib::forward(vs)...) + + switch (index) { + case B + 0: return TF_DISPATCH_AT(B + 0); + case B + 1: return TF_DISPATCH_AT(B + 1); + case B + 2: return TF_DISPATCH_AT(B + 2); + case B + 3: return TF_DISPATCH_AT(B + 3); + case B + 4: return TF_DISPATCH_AT(B + 4); + case B + 5: return TF_DISPATCH_AT(B + 5); + case B + 6: return TF_DISPATCH_AT(B + 6); + case B + 7: return TF_DISPATCH_AT(B + 7); + case B + 8: return TF_DISPATCH_AT(B + 8); + case B + 9: return TF_DISPATCH_AT(B + 9); + case B + 10: return TF_DISPATCH_AT(B + 10); + case B + 11: return TF_DISPATCH_AT(B + 11); + case B + 12: return TF_DISPATCH_AT(B + 12); + case B + 13: return TF_DISPATCH_AT(B + 13); + case B + 14: return TF_DISPATCH_AT(B + 14); + case B + 15: return TF_DISPATCH_AT(B + 15); + case B + 16: return TF_DISPATCH_AT(B + 16); + case B + 17: return TF_DISPATCH_AT(B + 17); + case B + 18: return TF_DISPATCH_AT(B + 18); + case B + 19: return TF_DISPATCH_AT(B + 19); + case B + 20: return TF_DISPATCH_AT(B + 20); + case B + 21: return TF_DISPATCH_AT(B + 21); + case B + 22: return TF_DISPATCH_AT(B + 22); + case B + 23: return TF_DISPATCH_AT(B + 23); + case B + 24: return TF_DISPATCH_AT(B + 24); + case B + 25: return TF_DISPATCH_AT(B + 25); + case B + 26: return TF_DISPATCH_AT(B + 26); + case B + 27: return TF_DISPATCH_AT(B + 27); + case B + 28: return TF_DISPATCH_AT(B + 28); + case B + 29: return TF_DISPATCH_AT(B + 29); + case B + 30: return TF_DISPATCH_AT(B + 30); + case B + 31: return TF_DISPATCH_AT(B + 31); + default: return TF_DEFAULT(B + 32); + } + +#undef TF_DEFAULT +#undef TF_DISPATCH_AT + } + }; +#else + template + inline static constexpr const T &at(const T &elem) noexcept { + return elem; + } + + template + inline static constexpr const lib::remove_all_extents_t &at( + const lib::array &elems, std::size_t i, Is... is) noexcept { + return at(elems[i], is...); + } + + template + inline static constexpr lib::array, sizeof...(Fs) + 1> + make_farray(F &&f, Fs &&... fs) { + return {{lib::forward(f), lib::forward(fs)...}}; + } + + template + struct make_fmatrix_impl { + + template + inline static constexpr dispatch_result_t dispatch( + F &&f, Vs &&... vs) { + using Expected = dispatch_result_t; + using Actual = decltype(lib::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + +#ifdef TF_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto impl(lib::index_sequence) { + return &dispatch; + } + + template + inline static constexpr auto impl(Is, + lib::index_sequence, + Ls... ls) { + return make_farray(impl(lib::push_back_t{}, ls...)...); + } +#else + template + struct impl; + + template + struct impl> { + inline constexpr AUTO operator()() const + AUTO_RETURN(&dispatch) + }; + + template + struct impl, Ls...> { + inline constexpr AUTO operator()() const + AUTO_RETURN( + make_farray(impl, Ls...>{}()...)) + }; +#endif + }; + +#ifdef TF_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto make_fmatrix() { + return make_fmatrix_impl::impl( + lib::index_sequence<>{}, + lib::make_index_sequence::size()>{}...); + } +#else + template + inline static constexpr AUTO make_fmatrix() + AUTO_RETURN( + typename make_fmatrix_impl::template impl< + lib::index_sequence<>, + lib::make_index_sequence::size()>...>{}()) +#endif + + template + struct make_fdiagonal_impl { + template + inline static constexpr dispatch_result_t dispatch( + F &&f, Vs &&... vs) { + using Expected = dispatch_result_t; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + + template + inline static constexpr AUTO impl(lib::index_sequence) + AUTO_RETURN(make_farray(&dispatch...)) + }; + + template + inline static constexpr auto make_fdiagonal() + -> decltype(make_fdiagonal_impl::impl( + lib::make_index_sequence::size()>{})) { + static_assert(lib::all<(lib::decay_t::size() == + lib::decay_t::size())...>::value, + "all of the variants must be the same size."); + return make_fdiagonal_impl::impl( + lib::make_index_sequence::size()>{}); + } +#endif + }; + +#if !defined(TF_VARIANT_SWITCH_VISIT) && \ + (!defined(_MSC_VER) || _MSC_VER >= 1910) + template + using fmatrix_t = decltype(base::make_fmatrix()); + + template + struct fmatrix { + static constexpr fmatrix_t value = + base::make_fmatrix(); + }; + + template + constexpr fmatrix_t fmatrix::value; + + template + using fdiagonal_t = decltype(base::make_fdiagonal()); + + template + struct fdiagonal { + static constexpr fdiagonal_t value = + base::make_fdiagonal(); + }; + + template + constexpr fdiagonal_t fdiagonal::value; +#endif + + struct alt { + template + inline static constexpr DECLTYPE_AUTO visit_alt(Visitor &&visitor, + Vs &&... vs) +#ifdef TF_VARIANT_SWITCH_VISIT + DECLTYPE_AUTO_RETURN( + base::dispatcher< + true, + base::dispatch_result_t(vs)))...>>:: + template dispatch<0>(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#elif !defined(_MSC_VER) || _MSC_VER >= 1910 + DECLTYPE_AUTO_RETURN(base::at( + fmatrix(vs)))...>::value, + vs.index()...)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#else + DECLTYPE_AUTO_RETURN(base::at( + base::make_fmatrix(vs)))...>(), + vs.index()...)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#endif + + template + inline static constexpr DECLTYPE_AUTO visit_alt_at(std::size_t index, + Visitor &&visitor, + Vs &&... vs) +#ifdef TF_VARIANT_SWITCH_VISIT + DECLTYPE_AUTO_RETURN( + base::dispatcher< + true, + base::dispatch_result_t(vs)))...>>:: + template dispatch_at<0>(index, + lib::forward(visitor), + as_base(lib::forward(vs))...)) +#elif !defined(_MSC_VER) || _MSC_VER >= 1910 + DECLTYPE_AUTO_RETURN(base::at( + fdiagonal(vs)))...>::value, + index)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#else + DECLTYPE_AUTO_RETURN(base::at( + base::make_fdiagonal(vs)))...>(), + index)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#endif + }; + + struct variant { + private: + template + struct visitor { + template + inline static constexpr bool does_not_handle() { + return lib::is_invocable::value; + } + }; + + template + struct visit_exhaustiveness_check { + static_assert(visitor::template does_not_handle(), + "`visit` requires the visitor to be exhaustive."); + + inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor, + Values &&... values) + DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward(visitor), + lib::forward(values)...)) + }; + + template + struct value_visitor { + Visitor &&visitor_; + + template + inline constexpr DECLTYPE_AUTO operator()(Alts &&... alts) const + DECLTYPE_AUTO_RETURN( + visit_exhaustiveness_check< + Visitor, + decltype((lib::forward(alts).value))...>:: + invoke(lib::forward(visitor_), + lib::forward(alts).value...)) + }; + + template + inline static constexpr AUTO make_value_visitor(Visitor &&visitor) + AUTO_RETURN(value_visitor{lib::forward(visitor)}) + + public: + template + inline static constexpr DECLTYPE_AUTO visit_alt(Visitor &&visitor, + Vs &&... vs) + DECLTYPE_AUTO_RETURN(alt::visit_alt(lib::forward(visitor), + lib::forward(vs).impl_...)) + + template + inline static constexpr DECLTYPE_AUTO visit_alt_at(std::size_t index, + Visitor &&visitor, + Vs &&... vs) + DECLTYPE_AUTO_RETURN( + alt::visit_alt_at(index, + lib::forward(visitor), + lib::forward(vs).impl_...)) + + template + inline static constexpr DECLTYPE_AUTO visit_value(Visitor &&visitor, + Vs &&... vs) + DECLTYPE_AUTO_RETURN( + visit_alt(make_value_visitor(lib::forward(visitor)), + lib::forward(vs)...)) + + template + inline static constexpr DECLTYPE_AUTO visit_value_at(std::size_t index, + Visitor &&visitor, + Vs &&... vs) + DECLTYPE_AUTO_RETURN( + visit_alt_at(index, + make_value_visitor(lib::forward(visitor)), + lib::forward(vs)...)) + }; + + } // namespace visitation + + template + struct alt { + using value_type = T; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4244) +#endif + template + inline explicit constexpr alt(in_place_t, Args &&... args) + : value(lib::forward(args)...) {} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + T value; + }; + + template + union recursive_union; + + template + union recursive_union {}; + +#define TF_VARIANT_RECURSIVE_UNION(destructible_trait, destructor) \ + template \ + union recursive_union { \ + public: \ + inline explicit constexpr recursive_union(valueless_t) noexcept \ + : dummy_{} {} \ + \ + template \ + inline explicit constexpr recursive_union(in_place_index_t<0>, \ + Args &&... args) \ + : head_(in_place_t{}, lib::forward(args)...) {} \ + \ + template \ + inline explicit constexpr recursive_union(in_place_index_t, \ + Args &&... args) \ + : tail_(in_place_index_t{}, lib::forward(args)...) {} \ + \ + recursive_union(const recursive_union &) = default; \ + recursive_union(recursive_union &&) = default; \ + \ + destructor \ + \ + recursive_union &operator=(const recursive_union &) = default; \ + recursive_union &operator=(recursive_union &&) = default; \ + \ + private: \ + char dummy_; \ + alt head_; \ + recursive_union tail_; \ + \ + friend struct access::recursive_union; \ + } + + TF_VARIANT_RECURSIVE_UNION(Trait::TriviallyAvailable, + ~recursive_union() = default;); + TF_VARIANT_RECURSIVE_UNION(Trait::Available, + ~recursive_union() {}); + TF_VARIANT_RECURSIVE_UNION(Trait::Unavailable, + ~recursive_union() = delete;); + +#undef TF_VARIANT_RECURSIVE_UNION + + template + using index_t = typename std::conditional< + sizeof...(Ts) < (std::numeric_limits::max)(), + unsigned char, + typename std::conditional< + sizeof...(Ts) < (std::numeric_limits::max)(), + unsigned short, + unsigned int>::type + >::type; + + template + class base { + public: + inline explicit constexpr base(valueless_t tag) noexcept + : data_(tag), index_(static_cast>(-1)) {} + + template + inline explicit constexpr base(in_place_index_t, Args &&... args) + : data_(in_place_index_t{}, lib::forward(args)...), + index_(I) {} + + inline constexpr bool valueless_by_exception() const noexcept { + return index_ == static_cast>(-1); + } + + inline constexpr std::size_t index() const noexcept { + return valueless_by_exception() ? variant_npos : index_; + } + + protected: + using data_t = recursive_union; + + friend inline constexpr base &as_base(base &b) { return b; } + friend inline constexpr const base &as_base(const base &b) { return b; } + friend inline constexpr base &&as_base(base &&b) { return lib::move(b); } + friend inline constexpr const base &&as_base(const base &&b) { return lib::move(b); } + + friend inline constexpr data_t &data(base &b) { return b.data_; } + friend inline constexpr const data_t &data(const base &b) { return b.data_; } + friend inline constexpr data_t &&data(base &&b) { return lib::move(b).data_; } + friend inline constexpr const data_t &&data(const base &&b) { return lib::move(b).data_; } + + inline static constexpr std::size_t size() { return sizeof...(Ts); } + + data_t data_; + index_t index_; + + friend struct access::base; + friend struct visitation::base; + }; + + struct dtor { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template + inline void operator()(Alt &alt) const noexcept { alt.~Alt(); } +#ifdef _MSC_VER +#pragma warning(pop) +#endif + }; + +#if !defined(_MSC_VER) || _MSC_VER >= 1910 +#define TF_INHERITING_CTOR(type, base) using base::base; +#else +#define TF_INHERITING_CTOR(type, base) \ + template \ + inline explicit constexpr type(Args &&... args) \ + : base(lib::forward(args)...) {} +#endif + + template + class destructor; + +#define TF_VARIANT_DESTRUCTOR(destructible_trait, definition, destroy) \ + template \ + class destructor, destructible_trait> \ + : public base { \ + using super = base; \ + \ + public: \ + TF_INHERITING_CTOR(destructor, super) \ + using super::operator=; \ + \ + destructor(const destructor &) = default; \ + destructor(destructor &&) = default; \ + definition \ + destructor &operator=(const destructor &) = default; \ + destructor &operator=(destructor &&) = default; \ + \ + protected: \ + destroy \ + } + + TF_VARIANT_DESTRUCTOR( + Trait::TriviallyAvailable, + ~destructor() = default;, + inline void destroy() noexcept { + this->index_ = static_cast>(-1); + }); + + TF_VARIANT_DESTRUCTOR( + Trait::Available, + ~destructor() { destroy(); }, + inline void destroy() noexcept { + if (!this->valueless_by_exception()) { + visitation::alt::visit_alt(dtor{}, *this); + } + this->index_ = static_cast>(-1); + }); + + TF_VARIANT_DESTRUCTOR( + Trait::Unavailable, + ~destructor() = delete;, + inline void destroy() noexcept = delete;); + +#undef TF_VARIANT_DESTRUCTOR + + template + class constructor : public destructor { + using super = destructor; + + public: + TF_INHERITING_CTOR(constructor, super) + using super::operator=; + + protected: +#ifndef TF_GENERIC_LAMBDAS + struct ctor { + template + inline void operator()(LhsAlt &lhs_alt, RhsAlt &&rhs_alt) const { + constructor::construct_alt(lhs_alt, + lib::forward(rhs_alt).value); + } + }; +#endif + + template + inline static T &construct_alt(alt &a, Args &&... args) { + auto *result = ::new (static_cast(lib::addressof(a))) + alt(in_place_t{}, lib::forward(args)...); + return result->value; + } + + template + inline static void generic_construct(constructor &lhs, Rhs &&rhs) { + lhs.destroy(); + if (!rhs.valueless_by_exception()) { + visitation::alt::visit_alt_at( + rhs.index(), +#ifdef TF_GENERIC_LAMBDAS + [](auto &lhs_alt, auto &&rhs_alt) { + constructor::construct_alt( + lhs_alt, lib::forward(rhs_alt).value); + } +#else + ctor{} +#endif + , + lhs, + lib::forward(rhs)); + lhs.index_ = rhs.index_; + } + } + }; + + template + class move_constructor; + +#define TF_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, definition) \ + template \ + class move_constructor, move_constructible_trait> \ + : public constructor> { \ + using super = constructor>; \ + \ + public: \ + TF_INHERITING_CTOR(move_constructor, super) \ + using super::operator=; \ + \ + move_constructor(const move_constructor &) = default; \ + definition \ + ~move_constructor() = default; \ + move_constructor &operator=(const move_constructor &) = default; \ + move_constructor &operator=(move_constructor &&) = default; \ + } + + TF_VARIANT_MOVE_CONSTRUCTOR( + Trait::TriviallyAvailable, + move_constructor(move_constructor &&that) = default;); + + TF_VARIANT_MOVE_CONSTRUCTOR( + Trait::Available, + move_constructor(move_constructor &&that) noexcept( + lib::all::value...>::value) + : move_constructor(valueless_t{}) { + this->generic_construct(*this, lib::move(that)); + }); + + TF_VARIANT_MOVE_CONSTRUCTOR( + Trait::Unavailable, + move_constructor(move_constructor &&) = delete;); + +#undef TF_VARIANT_MOVE_CONSTRUCTOR + + template + class copy_constructor; + +#define TF_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, definition) \ + template \ + class copy_constructor, copy_constructible_trait> \ + : public move_constructor> { \ + using super = move_constructor>; \ + \ + public: \ + TF_INHERITING_CTOR(copy_constructor, super) \ + using super::operator=; \ + \ + definition \ + copy_constructor(copy_constructor &&) = default; \ + ~copy_constructor() = default; \ + copy_constructor &operator=(const copy_constructor &) = default; \ + copy_constructor &operator=(copy_constructor &&) = default; \ + } + + TF_VARIANT_COPY_CONSTRUCTOR( + Trait::TriviallyAvailable, + copy_constructor(const copy_constructor &that) = default;); + + TF_VARIANT_COPY_CONSTRUCTOR( + Trait::Available, + copy_constructor(const copy_constructor &that) + : copy_constructor(valueless_t{}) { + this->generic_construct(*this, that); + }); + + TF_VARIANT_COPY_CONSTRUCTOR( + Trait::Unavailable, + copy_constructor(const copy_constructor &) = delete;); + +#undef TF_VARIANT_COPY_CONSTRUCTOR + + template + class assignment : public copy_constructor { + using super = copy_constructor; + + public: + TF_INHERITING_CTOR(assignment, super) + using super::operator=; + + template + inline /* auto & */ auto emplace(Args &&... args) + -> decltype(this->construct_alt(access::base::get_alt(*this), + lib::forward(args)...)) { + this->destroy(); + auto &result = this->construct_alt(access::base::get_alt(*this), + lib::forward(args)...); + this->index_ = I; + return result; + } + + protected: +#ifndef TF_GENERIC_LAMBDAS + template + struct assigner { + template + inline void operator()(ThisAlt &this_alt, ThatAlt &&that_alt) const { + self->assign_alt(this_alt, lib::forward(that_alt).value); + } + assignment *self; + }; +#endif + + template + inline void assign_alt(alt &a, Arg &&arg) { + if (this->index() == I) { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4244) +#endif + a.value = lib::forward(arg); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + } else { + struct { + void operator()(std::true_type) const { + this_->emplace(lib::forward(arg_)); + } + void operator()(std::false_type) const { + this_->emplace(T(lib::forward(arg_))); + } + assignment *this_; + Arg &&arg_; + } impl{this, lib::forward(arg)}; + impl(lib::bool_constant< + std::is_nothrow_constructible::value || + !std::is_nothrow_move_constructible::value>{}); + } + } + + template + inline void generic_assign(That &&that) { + if (this->valueless_by_exception() && that.valueless_by_exception()) { + // do nothing. + } else if (that.valueless_by_exception()) { + this->destroy(); + } else { + visitation::alt::visit_alt_at( + that.index(), +#ifdef TF_GENERIC_LAMBDAS + [this](auto &this_alt, auto &&that_alt) { + this->assign_alt( + this_alt, lib::forward(that_alt).value); + } +#else + assigner{this} +#endif + , + *this, + lib::forward(that)); + } + } + }; + + template + class move_assignment; + +#define TF_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, definition) \ + template \ + class move_assignment, move_assignable_trait> \ + : public assignment> { \ + using super = assignment>; \ + \ + public: \ + TF_INHERITING_CTOR(move_assignment, super) \ + using super::operator=; \ + \ + move_assignment(const move_assignment &) = default; \ + move_assignment(move_assignment &&) = default; \ + ~move_assignment() = default; \ + move_assignment &operator=(const move_assignment &) = default; \ + definition \ + } + + TF_VARIANT_MOVE_ASSIGNMENT( + Trait::TriviallyAvailable, + move_assignment &operator=(move_assignment &&that) = default;); + + TF_VARIANT_MOVE_ASSIGNMENT( + Trait::Available, + move_assignment & + operator=(move_assignment &&that) noexcept( + lib::all<(std::is_nothrow_move_constructible::value && + std::is_nothrow_move_assignable::value)...>::value) { + this->generic_assign(lib::move(that)); + return *this; + }); + + TF_VARIANT_MOVE_ASSIGNMENT( + Trait::Unavailable, + move_assignment &operator=(move_assignment &&) = delete;); + +#undef TF_VARIANT_MOVE_ASSIGNMENT + + template + class copy_assignment; + +#define TF_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, definition) \ + template \ + class copy_assignment, copy_assignable_trait> \ + : public move_assignment> { \ + using super = move_assignment>; \ + \ + public: \ + TF_INHERITING_CTOR(copy_assignment, super) \ + using super::operator=; \ + \ + copy_assignment(const copy_assignment &) = default; \ + copy_assignment(copy_assignment &&) = default; \ + ~copy_assignment() = default; \ + definition \ + copy_assignment &operator=(copy_assignment &&) = default; \ + } + + TF_VARIANT_COPY_ASSIGNMENT( + Trait::TriviallyAvailable, + copy_assignment &operator=(const copy_assignment &that) = default;); + + TF_VARIANT_COPY_ASSIGNMENT( + Trait::Available, + copy_assignment &operator=(const copy_assignment &that) { + this->generic_assign(that); + return *this; + }); + + TF_VARIANT_COPY_ASSIGNMENT( + Trait::Unavailable, + copy_assignment &operator=(const copy_assignment &) = delete;); + +#undef TF_VARIANT_COPY_ASSIGNMENT + + template + class impl : public copy_assignment> { + using super = copy_assignment>; + + public: + TF_INHERITING_CTOR(impl, super) + using super::operator=; + + impl(const impl&) = default; + impl(impl&&) = default; + ~impl() = default; + impl &operator=(const impl &) = default; + impl &operator=(impl &&) = default; + + template + inline void assign(Arg &&arg) { + this->assign_alt(access::base::get_alt(*this), + lib::forward(arg)); + } + + inline void swap(impl &that) { + if (this->valueless_by_exception() && that.valueless_by_exception()) { + // do nothing. + } else if (this->index() == that.index()) { + visitation::alt::visit_alt_at(this->index(), +#ifdef TF_GENERIC_LAMBDAS + [](auto &this_alt, auto &that_alt) { + using std::swap; + swap(this_alt.value, + that_alt.value); + } +#else + swapper{} +#endif + , + *this, + that); + } else { + impl *lhs = this; + impl *rhs = lib::addressof(that); + if (lhs->move_nothrow() && !rhs->move_nothrow()) { + std::swap(lhs, rhs); + } + impl tmp(lib::move(*rhs)); +#ifdef TF_EXCEPTIONS + // EXTENSION: When the move construction of `lhs` into `rhs` throws + // and `tmp` is nothrow move constructible then we move `tmp` back + // into `rhs` and provide the strong exception safety guarantee. + try { + this->generic_construct(*rhs, lib::move(*lhs)); + } catch (...) { + if (tmp.move_nothrow()) { + this->generic_construct(*rhs, lib::move(tmp)); + } + throw; + } +#else + this->generic_construct(*rhs, lib::move(*lhs)); +#endif + this->generic_construct(*lhs, lib::move(tmp)); + } + } + + private: +#ifndef TF_GENERIC_LAMBDAS + struct swapper { + template + inline void operator()(ThisAlt &this_alt, ThatAlt &that_alt) const { + using std::swap; + swap(this_alt.value, that_alt.value); + } + }; +#endif + + inline constexpr bool move_nothrow() const { + return this->valueless_by_exception() || + lib::array{ + {std::is_nothrow_move_constructible::value...} + }[this->index()]; + } + }; + +#undef TF_INHERITING_CTOR + + template + struct is_non_narrowing_convertible { + template + static std::true_type test(T(&&)[1]); + + template + static auto impl(int) -> decltype(test({std::declval()})); + + template + static auto impl(...) -> std::false_type; + + static constexpr bool value = decltype(impl(0))::value; + }; + + template ::value, + typename = void> + struct overload_leaf {}; + + template + struct overload_leaf { + using impl = lib::size_constant (*)(T); + operator impl() const { return nullptr; }; + }; + + template + struct overload_leaf< + Arg, + I, + T, + true +#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ >= 5 + , + lib::enable_if_t< + std::is_same, bool>::value + ? std::is_same, bool>::value + : is_non_narrowing_convertible::value> +#endif + > { + using impl = lib::size_constant (*)(T); + operator impl() const { return nullptr; }; + }; + + template + struct overload_impl { + private: + template + struct impl; + + template + struct impl> : overload_leaf... {}; + + public: + using type = impl>; + }; + + template + using overload = typename overload_impl::type; + + template + using best_match = lib::invoke_result_t, Arg>; + + template + struct is_in_place_index : std::false_type {}; + + template + struct is_in_place_index> : std::true_type {}; + + template + struct is_in_place_type : std::false_type {}; + + template + struct is_in_place_type> : std::true_type {}; + + } // detail + + template + class variant { + static_assert(0 < sizeof...(Ts), + "variant must consist of at least one alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have an array type as an alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have a reference type as an alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have a void type as an alternative."); + + public: + template < + typename Front = lib::type_pack_element_t<0, Ts...>, + lib::enable_if_t::value, int> = 0> + inline constexpr variant() noexcept( + std::is_nothrow_default_constructible::value) + : impl_(in_place_index_t<0>{}) {} + + variant(const variant &) = default; + variant(variant &&) = default; + + template < + typename Arg, + typename Decayed = lib::decay_t, + lib::enable_if_t::value, int> = 0, + lib::enable_if_t::value, int> = 0, + lib::enable_if_t::value, int> = 0, + std::size_t I = detail::best_match::value, + typename T = lib::type_pack_element_t, + lib::enable_if_t::value, int> = 0> + inline constexpr variant(Arg &&arg) noexcept( + std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(arg)) {} + + template < + std::size_t I, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t::value, int> = 0> + inline explicit constexpr variant( + in_place_index_t, + Args &&... args) noexcept(std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(args)...) {} + + template < + std::size_t I, + typename Up, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t &, + Args...>::value, + int> = 0> + inline explicit constexpr variant( + in_place_index_t, + std::initializer_list il, + Args &&... args) noexcept(std:: + is_nothrow_constructible< + T, + std::initializer_list &, + Args...>::value) + : impl_(in_place_index_t{}, il, lib::forward(args)...) {} + + template < + typename T, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t::value, int> = 0> + inline explicit constexpr variant( + in_place_type_t, + Args &&... args) noexcept(std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(args)...) {} + + template < + typename T, + typename Up, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t &, + Args...>::value, + int> = 0> + inline explicit constexpr variant( + in_place_type_t, + std::initializer_list il, + Args &&... args) noexcept(std:: + is_nothrow_constructible< + T, + std::initializer_list &, + Args...>::value) + : impl_(in_place_index_t{}, il, lib::forward(args)...) {} + + ~variant() = default; + + variant &operator=(const variant &) = default; + variant &operator=(variant &&) = default; + + template , variant>::value, + int> = 0, + std::size_t I = detail::best_match::value, + typename T = lib::type_pack_element_t, + lib::enable_if_t<(std::is_assignable::value && + std::is_constructible::value), + int> = 0> + inline variant &operator=(Arg &&arg) noexcept( + (std::is_nothrow_assignable::value && + std::is_nothrow_constructible::value)) { + impl_.template assign(lib::forward(arg)); + return *this; + } + + template < + std::size_t I, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t::value, int> = 0> + inline T &emplace(Args &&... args) { + return impl_.template emplace(lib::forward(args)...); + } + + template < + std::size_t I, + typename Up, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t &, + Args...>::value, + int> = 0> + inline T &emplace(std::initializer_list il, Args &&... args) { + return impl_.template emplace(il, lib::forward(args)...); + } + + template < + typename T, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t::value, int> = 0> + inline T &emplace(Args &&... args) { + return impl_.template emplace(lib::forward(args)...); + } + + template < + typename T, + typename Up, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t &, + Args...>::value, + int> = 0> + inline T &emplace(std::initializer_list il, Args &&... args) { + return impl_.template emplace(il, lib::forward(args)...); + } + + inline constexpr bool valueless_by_exception() const noexcept { + return impl_.valueless_by_exception(); + } + + inline constexpr std::size_t index() const noexcept { + return impl_.index(); + } + + template , + Dummy>::value && + lib::dependent_type, + Dummy>::value)...>::value, + int> = 0> + inline void swap(variant &that) noexcept( + lib::all<(std::is_nothrow_move_constructible::value && + lib::is_nothrow_swappable::value)...>::value) { + impl_.swap(that.impl_); + } + + private: + detail::impl impl_; + + friend struct detail::access::variant; + friend struct detail::visitation::variant; + }; + + template + inline constexpr bool holds_alternative(const variant &v) noexcept { + return v.index() == I; + } + + template + inline constexpr bool holds_alternative(const variant &v) noexcept { + return holds_alternative::value>(v); + } + + namespace detail { + template + struct generic_get_impl { + constexpr generic_get_impl(int) noexcept {} + + constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN( + access::variant::get_alt(lib::forward(v)).value) + }; + + template + inline constexpr AUTO_REFREF generic_get(V &&v) + AUTO_REFREF_RETURN(generic_get_impl( + holds_alternative(v) ? 0 : (throw_bad_variant_access(), 0))( + lib::forward(v))) + } // namespace detail + + template + inline constexpr variant_alternative_t> &get( + variant &v) { + return detail::generic_get(v); + } + + template + inline constexpr variant_alternative_t> &&get( + variant &&v) { + return detail::generic_get(lib::move(v)); + } + + template + inline constexpr const variant_alternative_t> &get( + const variant &v) { + return detail::generic_get(v); + } + + template + inline constexpr const variant_alternative_t> &&get( + const variant &&v) { + return detail::generic_get(lib::move(v)); + } + + template + inline constexpr T &get(variant &v) { + return get::value>(v); + } + + template + inline constexpr T &&get(variant &&v) { + return get::value>(lib::move(v)); + } + + template + inline constexpr const T &get(const variant &v) { + return get::value>(v); + } + + template + inline constexpr const T &&get(const variant &&v) { + return get::value>(lib::move(v)); + } + + namespace detail { + + template + inline constexpr /* auto * */ AUTO generic_get_if(V *v) noexcept + AUTO_RETURN(v && holds_alternative(*v) + ? lib::addressof(access::variant::get_alt(*v).value) + : nullptr) + + } // namespace detail + + template + inline constexpr lib::add_pointer_t>> + get_if(variant *v) noexcept { + return detail::generic_get_if(v); + } + + template + inline constexpr lib::add_pointer_t< + const variant_alternative_t>> + get_if(const variant *v) noexcept { + return detail::generic_get_if(v); + } + + template + inline constexpr lib::add_pointer_t + get_if(variant *v) noexcept { + return get_if::value>(v); + } + + template + inline constexpr lib::add_pointer_t + get_if(const variant *v) noexcept { + return get_if::value>(v); + } + + namespace detail { + template + struct convert_to_bool { + template + inline constexpr bool operator()(Lhs &&lhs, Rhs &&rhs) const { + static_assert(std::is_convertible, + bool>::value, + "relational operators must return a type" + " implicitly convertible to bool"); + return lib::invoke( + RelOp{}, lib::forward(lhs), lib::forward(rhs)); + } + }; + } // namespace detail + + template + inline constexpr bool operator==(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using equal_to = detail::convert_to_bool; +#ifdef TF_CPP14_CONSTEXPR + if (lhs.index() != rhs.index()) return false; + if (lhs.valueless_by_exception()) return true; + return variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs); +#else + return lhs.index() == rhs.index() && + (lhs.valueless_by_exception() || + variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs)); +#endif + } + + template + inline constexpr bool operator!=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using not_equal_to = detail::convert_to_bool; +#ifdef TF_CPP14_CONSTEXPR + if (lhs.index() != rhs.index()) return true; + if (lhs.valueless_by_exception()) return false; + return variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs); +#else + return lhs.index() != rhs.index() || + (!lhs.valueless_by_exception() && + variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs)); +#endif + } + + template + inline constexpr bool operator<(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using less = detail::convert_to_bool; +#ifdef TF_CPP14_CONSTEXPR + if (rhs.valueless_by_exception()) return false; + if (lhs.valueless_by_exception()) return true; + if (lhs.index() < rhs.index()) return true; + if (lhs.index() > rhs.index()) return false; + return variant::visit_value_at(lhs.index(), less{}, lhs, rhs); +#else + return !rhs.valueless_by_exception() && + (lhs.valueless_by_exception() || lhs.index() < rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), less{}, lhs, rhs))); +#endif + } + + template + inline constexpr bool operator>(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using greater = detail::convert_to_bool; +#ifdef TF_CPP14_CONSTEXPR + if (lhs.valueless_by_exception()) return false; + if (rhs.valueless_by_exception()) return true; + if (lhs.index() > rhs.index()) return true; + if (lhs.index() < rhs.index()) return false; + return variant::visit_value_at(lhs.index(), greater{}, lhs, rhs); +#else + return !lhs.valueless_by_exception() && + (rhs.valueless_by_exception() || lhs.index() > rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), greater{}, lhs, rhs))); +#endif + } + + template + inline constexpr bool operator<=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using less_equal = detail::convert_to_bool; +#ifdef TF_CPP14_CONSTEXPR + if (lhs.valueless_by_exception()) return true; + if (rhs.valueless_by_exception()) return false; + if (lhs.index() < rhs.index()) return true; + if (lhs.index() > rhs.index()) return false; + return variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs); +#else + return lhs.valueless_by_exception() || + (!rhs.valueless_by_exception() && + (lhs.index() < rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs)))); +#endif + } + + template + inline constexpr bool operator>=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using greater_equal = detail::convert_to_bool; +#ifdef TF_CPP14_CONSTEXPR + if (rhs.valueless_by_exception()) return true; + if (lhs.valueless_by_exception()) return false; + if (lhs.index() > rhs.index()) return true; + if (lhs.index() < rhs.index()) return false; + return variant::visit_value_at(lhs.index(), greater_equal{}, lhs, rhs); +#else + return rhs.valueless_by_exception() || + (!lhs.valueless_by_exception() && + (lhs.index() > rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at( + lhs.index(), greater_equal{}, lhs, rhs)))); +#endif + } + + struct monostate {}; + + inline constexpr bool operator<(monostate, monostate) noexcept { + return false; + } + + inline constexpr bool operator>(monostate, monostate) noexcept { + return false; + } + + inline constexpr bool operator<=(monostate, monostate) noexcept { + return true; + } + + inline constexpr bool operator>=(monostate, monostate) noexcept { + return true; + } + + inline constexpr bool operator==(monostate, monostate) noexcept { + return true; + } + + inline constexpr bool operator!=(monostate, monostate) noexcept { + return false; + } + +#ifdef TF_CPP14_CONSTEXPR + namespace detail { + + inline constexpr bool any(std::initializer_list bs) { + for (bool b : bs) { + if (b) { + return true; + } + } + return false; + } + + } // namespace detail + + template + inline constexpr decltype(auto) visit(Visitor &&visitor, Vs &&... vs) { + return (!detail::any({vs.valueless_by_exception()...}) + ? (void)0 + : throw_bad_variant_access()), + detail::visitation::variant::visit_value( + lib::forward(visitor), lib::forward(vs)...); + } +#else + namespace detail { + + template + inline constexpr bool all_impl(const lib::array &bs, + std::size_t idx) { + return idx >= N || (bs[idx] && all_impl(bs, idx + 1)); + } + + template + inline constexpr bool all(const lib::array &bs) { + return all_impl(bs, 0); + } + + } // namespace detail + + template + inline constexpr DECLTYPE_AUTO visit(Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN( + (detail::all( + lib::array{{!vs.valueless_by_exception()...}}) + ? (void)0 + : throw_bad_variant_access()), + detail::visitation::variant::visit_value(lib::forward(visitor), + lib::forward(vs)...)) +#endif + + template + inline auto swap(variant &lhs, + variant &rhs) noexcept(noexcept(lhs.swap(rhs))) + -> decltype(lhs.swap(rhs)) { + lhs.swap(rhs); + } + + namespace detail { + + template + using enabled_type = T; + + namespace hash { + + template + constexpr bool meets_requirements() noexcept { + return std::is_copy_constructible::value && + std::is_move_constructible::value && + lib::is_invocable_r::value; + } + + template + constexpr bool is_enabled() noexcept { + using H = std::hash; + return meets_requirements() && + std::is_default_constructible::value && + std::is_copy_assignable::value && + std::is_move_assignable::value; + } + + } // namespace hash + + } // namespace detail + +#undef AUTO +#undef AUTO_RETURN + +#undef AUTO_REFREF +#undef AUTO_REFREF_RETURN + +#undef DECLTYPE_AUTO +#undef DECLTYPE_AUTO_RETURN + +}} // namespace tf::nstd + +namespace std { + + template + struct hash, + tf::nstd::lib::enable_if_t>()...>::value>>> { + using argument_type = tf::nstd::variant; + using result_type = std::size_t; + + inline result_type operator()(const argument_type &v) const { + using tf::nstd::detail::visitation::variant; + std::size_t result = + v.valueless_by_exception() + ? 299792458 // Random value chosen by the universe upon creation + : variant::visit_alt( +#ifdef TF_GENERIC_LAMBDAS + [](const auto &alt) { + using alt_type = tf::nstd::lib::decay_t; + using value_type = tf::nstd::lib::remove_const_t< + typename alt_type::value_type>; + return hash{}(alt.value); + } +#else + hasher{} +#endif + , + v); + return hash_combine(result, hash{}(v.index())); + } + + private: +#ifndef TF_GENERIC_LAMBDAS + struct hasher { + template + inline std::size_t operator()(const Alt &alt) const { + using alt_type = tf::nstd::lib::decay_t; + using value_type = + tf::nstd::lib::remove_const_t; + return hash{}(alt.value); + } + }; +#endif + + static std::size_t hash_combine(std::size_t lhs, std::size_t rhs) { + return lhs ^= rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2); + } + }; + + template <> + struct hash { + using argument_type = tf::nstd::monostate; + using result_type = std::size_t; + + inline result_type operator()(const argument_type &) const noexcept { + return 66740831; // return a fundamentally attractive random value. + } + }; + +} // namespace std + + + diff --git a/bundled/cpp-taskflow/include/taskflow/taskflow.hpp b/bundled/cpp-taskflow/include/taskflow/taskflow.hpp new file mode 100644 index 0000000000..192f8a8cfd --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/taskflow.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "core/executor.hpp" + +namespace tf { + + +} // end of namespace tf. --------------------------------------------------- + + + + + diff --git a/bundled/cpp-taskflow/include/taskflow/utility/object_pool.hpp b/bundled/cpp-taskflow/include/taskflow/utility/object_pool.hpp new file mode 100644 index 0000000000..4272e339d5 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/utility/object_pool.hpp @@ -0,0 +1,776 @@ +// 2020/03/13 - modified by Tsung-Wei Huang +// - fixed bug in aligning memory +// +// 2020/02/02 - modified by Tsung-Wei Huang +// - new implementation motivated by Hoard +// +// 2019/07/10 - modified by Tsung-Wei Huang +// - replace raw pointer with smart pointer +// +// 2019/06/13 - created by Tsung-Wei Huang +// - implemented an object pool class + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace tf { + +#define TF_ENABLE_POOLABLE_ON_THIS \ + template friend class ObjectPool; \ + void* _object_pool_block; + +// Class: ObjectPool +// +// The class implements an efficient thread-safe object pool motivated +// by the Hoard memory allocator algorithm. +// Different from the normal memory allocator, object pool allocates +// only one object at a time. +// +// Internall, we use the following variables to maintain blocks and heaps: +// X: size in byte of a item slot +// M: number of items per block +// F: emptiness threshold +// B: number of bins per local heap (bin[B-1] is the full list) +// W: number of items per bin +// K: shrinkness constant +// +// Example scenario 1: +// M = 30 +// F = 4 +// W = (30+4-1)/4 = 8 +// +// b0: 0, 1, 2, 3, 4, 5, 6, 7 +// b1: 8, 9, 10, 11, 12, 13, 14, 15 +// b2: 16, 17, 18, 19, 20, 21, 22, 23 +// b3: 24, 25, 26, 27, 28, 29 +// b4: 30 (anything equal to M) +// +// Example scenario 2: +// M = 32 +// F = 4 +// W = (32+4-1)/4 = 8 +// b0: 0, 1, 2, 3, 4, 5, 6, 7 +// b1: 8, 9, 10, 11, 12, 13, 14, 15 +// b2: 16, 17, 18, 19, 20, 21, 22, 23 +// b3: 24, 25, 26, 27, 28, 29, 30, 31 +// b4: 32 (anything equal to M) +// +template +class ObjectPool { + + // the data column must be sufficient to hold the pointer in freelist + constexpr static size_t X = std::max(sizeof(T*), sizeof(T)); + //constexpr static size_t X = sizeof(long double) + std::max(sizeof(T*), sizeof(T)); + //constexpr static size_t M = (S - offsetof(Block, data)) / X; + constexpr static size_t M = S / X; + constexpr static size_t F = 4; + constexpr static size_t B = F + 1; + constexpr static size_t W = (M + F - 1) / F; + constexpr static size_t K = 4; + + static_assert( + S && (!(S & (S-1))), "block size S must be a power of two" + ); + + static_assert( + M >= 128, "block size S must be larger enough to pool at least 128 objects" + ); + + struct Blocklist { + Blocklist* prev; + Blocklist* next; + }; + + class GlobalHeap { + friend class ObjectPool; + std::mutex mutex; + Blocklist list; + }; + + class LocalHeap { + friend class ObjectPool; + std::mutex mutex; + Blocklist lists[B]; + size_t u {0}; + size_t a {0}; + }; + + struct Block { + LocalHeap* heap; + Blocklist list_node; + size_t i; + size_t u; + T* top; + // long double padding; + char data[S]; + }; + + public: + + /** + @brief constructs an object pool from a number of anticipated threads + */ + explicit ObjectPool(unsigned = std::thread::hardware_concurrency()); + + /** + @brief destructs the object pool + */ + ~ObjectPool(); + + /** + @brief acquires a pointer to a object constructed from a given argument list + */ + template + T* animate(ArgsT&&... args); + + /** + @brief recycles a object pointed by @c ptr and destroys it + */ + void recycle(T* ptr); + + size_t num_bins_per_local_heap() const; + size_t num_objects_per_bin() const; + size_t num_objects_per_block() const; + size_t num_available_objects() const; + size_t num_allocated_objects() const; + size_t capacity() const; + size_t num_local_heaps() const; + size_t num_global_heaps() const; + size_t num_heaps() const; + + float emptiness_threshold() const; + + private: + + const size_t _lheap_mask; + + GlobalHeap _gheap; + + std::vector _lheaps; + + LocalHeap& _this_heap(); + + constexpr unsigned _next_power_of_two(unsigned n) const; + + template + constexpr size_t _offset_in_class(const Q P::*member) const; + + template + constexpr P* _parent_class_of(Q*, const Q P::*member); + + template + constexpr P* _parent_class_of(const Q*, const Q P::*member) const; + + constexpr Block* _block_of(Blocklist*); + constexpr Block* _block_of(const Blocklist*) const; + + size_t _bin(size_t) const; + + T* _allocate(Block*); + + void _deallocate(Block*, T*); + void _blocklist_init_head(Blocklist*); + void _blocklist_add_impl(Blocklist*, Blocklist*, Blocklist*); + void _blocklist_push_front(Blocklist*, Blocklist*); + void _blocklist_push_back(Blocklist*, Blocklist*); + void _blocklist_del_impl(Blocklist*, Blocklist*); + void _blocklist_del(Blocklist*); + void _blocklist_replace(Blocklist*, Blocklist*); + void _blocklist_move_front(Blocklist*, Blocklist*); + void _blocklist_move_back(Blocklist*, Blocklist*); + bool _blocklist_is_first(const Blocklist*, const Blocklist*); + bool _blocklist_is_last(const Blocklist*, const Blocklist*); + bool _blocklist_is_empty(const Blocklist*); + bool _blocklist_is_singular(const Blocklist*); + + template + void _for_each_block_safe(Blocklist*, C&&); + + template + void _for_each_block(Blocklist*, C&&); + +}; + +// ---------------------------------------------------------------------------- +// ObjectPool definition +// ---------------------------------------------------------------------------- + +// Constructor +template +ObjectPool::ObjectPool(unsigned t) : + //_heap_mask {(_next_power_of_two(t) << 1) - 1u}, + //_heap_mask { _next_power_of_two(t<<1) - 1u }, + //_heap_mask {(t << 1) - 1}, + _lheap_mask { _next_power_of_two((t+1) << 1) - 1 }, + _lheaps { _lheap_mask + 1 } { + + _blocklist_init_head(&_gheap.list); + + for(auto& h : _lheaps) { + for(size_t i=0; i +ObjectPool::~ObjectPool() { + + // clear local heaps + for(auto& h : _lheaps) { + for(size_t i=0; i +size_t ObjectPool::num_bins_per_local_heap() const { + return B; +} + +// Function: num_objects_per_bin +template +size_t ObjectPool::num_objects_per_bin() const { + return W; +} + +// Function: num_objects_per_block +template +size_t ObjectPool::num_objects_per_block() const { + return M; +} + +// Function: emptiness_threshold +template +float ObjectPool::emptiness_threshold() const { + return 1.0f/F; +} + +// Function: num_global_heaps +template +size_t ObjectPool::num_global_heaps() const { + return 1; +} + +// Function: num_lheaps +template +size_t ObjectPool::num_local_heaps() const { + return _lheaps.size(); +} + +// Function: num_heaps +template +size_t ObjectPool::num_heaps() const { + return _lheaps.size() + 1; +} + +// Function: capacity +template +size_t ObjectPool::capacity() const { + + size_t n = 0; + + // global heap + for(auto p=_gheap.list.next; p!=&_gheap.list; p=p->next) { + n += M; + }; + + // local heap + for(auto& h : _lheaps) { + n += h.a; + } + + return n; +} + +// Function: num_available_objects +template +size_t ObjectPool::num_available_objects() const { + + size_t n = 0; + + // global heap + for(auto p=_gheap.list.next; p!=&_gheap.list; p=p->next) { + n += (M - _block_of(p)->u); + }; + + // local heap + for(auto& h : _lheaps) { + n += (h.a - h.u); + } + return n; +} + +// Function: num_allocated_objects +template +size_t ObjectPool::num_allocated_objects() const { + + size_t n = 0; + + // global heap + for(auto p=_gheap.list.next; p!=&_gheap.list; p=p->next) { + n += _block_of(p)->u; + }; + + // local heap + for(auto& h : _lheaps) { + n += h.u; + } + return n; +} + +// Function: _bin +template +size_t ObjectPool::_bin(size_t u) const { + return u == M ? F : u/W; +} + +// Function: _offset_in_class +template +template +constexpr size_t ObjectPool::_offset_in_class( + const Q P::*member) const { + return (size_t) &( reinterpret_cast(0)->*member); +} + +// C macro: parent_class_of(list_pointer, Block, list) +// C++: parent_class_of(list_pointer, &Block::list) +template +template +constexpr P* ObjectPool::_parent_class_of( + Q* ptr, const Q P::*member +) { + return (P*)( (char*)ptr - _offset_in_class(member)); +} + +// Function: _parent_class_of +template +template +constexpr P* ObjectPool::_parent_class_of( + const Q* ptr, const Q P::*member +) const { + return (P*)( (char*)ptr - _offset_in_class(member)); +} + +// Function: _block_of +template +constexpr typename ObjectPool::Block* +ObjectPool::_block_of(Blocklist* list) { + return _parent_class_of(list, &Block::list_node); +} + +// Function: _block_of +template +constexpr typename ObjectPool::Block* +ObjectPool::_block_of(const Blocklist* list) const { + return _parent_class_of(list, &Block::list_node); +} + +// Procedure: initialize a list head +template +void ObjectPool::_blocklist_init_head(Blocklist *list) { + list->next = list; + list->prev = list; +} + +// Procedure: _blocklist_add_impl +// Insert a new entry between two known consecutive entries. +// +// This is only for internal list manipulation where we know +// the prev/next entries already! +template +void ObjectPool::_blocklist_add_impl( + Blocklist *curr, Blocklist *prev, Blocklist *next +) { + next->prev = curr; + curr->next = next; + curr->prev = prev; + prev->next = curr; +} + +// list_push_front - add a new entry +// @curr: curr entry to be added +// @head: list head to add it after +// +// Insert a new entry after the specified head. +// This is good for implementing stacks. +// +template +void ObjectPool::_blocklist_push_front( + Blocklist *curr, Blocklist *head +) { + _blocklist_add_impl(curr, head, head->next); +} + +// list_add_tail - add a new entry +// @curr: curr entry to be added +// @head: list head to add it before +// +// Insert a new entry before the specified head. +// This is useful for implementing queues. +// +template +void ObjectPool::_blocklist_push_back( + Blocklist *curr, Blocklist *head +) { + _blocklist_add_impl(curr, head->prev, head); +} + +// Delete a list entry by making the prev/next entries +// point to each other. +// +// This is only for internal list manipulation where we know +// the prev/next entries already! +// +template +void ObjectPool::_blocklist_del_impl( + Blocklist * prev, Blocklist * next +) { + next->prev = prev; + prev->next = next; +} + +// _blocklist_del - deletes entry from list. +// @entry: the element to delete from the list. +// Note: list_empty() on entry does not return true after this, the entry is +// in an undefined state. +template +void ObjectPool::_blocklist_del(Blocklist *entry) { + _blocklist_del_impl(entry->prev, entry->next); + entry->next = nullptr; + entry->prev = nullptr; +} + +// list_replace - replace old entry by new one +// @old : the element to be replaced +// @curr : the new element to insert +// +// If @old was empty, it will be overwritten. +template +void ObjectPool::_blocklist_replace( + Blocklist *old, Blocklist *curr +) { + curr->next = old->next; + curr->next->prev = curr; + curr->prev = old->prev; + curr->prev->next = curr; +} + +// list_move - delete from one list and add as another's head +// @list: the entry to move +// @head: the head that will precede our entry +template +void ObjectPool::_blocklist_move_front( + Blocklist *list, Blocklist *head +) { + _blocklist_del_impl(list->prev, list->next); + _blocklist_push_front(list, head); +} + +// list_move_tail - delete from one list and add as another's tail +// @list: the entry to move +// @head: the head that will follow our entry +template +void ObjectPool::_blocklist_move_back( + Blocklist *list, Blocklist *head +) { + _blocklist_del_impl(list->prev, list->next); + _blocklist_push_back(list, head); +} + +// list_is_first - tests whether @list is the last entry in list @head +// @list: the entry to test +// @head: the head of the list +template +bool ObjectPool::_blocklist_is_first( + const Blocklist *list, const Blocklist *head +) { + return list->prev == head; +} + +// list_is_last - tests whether @list is the last entry in list @head +// @list: the entry to test +// @head: the head of the list +template +bool ObjectPool::_blocklist_is_last( + const Blocklist *list, const Blocklist *head +) { + return list->next == head; +} + +// list_empty - tests whether a list is empty +// @head: the list to test. +template +bool ObjectPool::_blocklist_is_empty(const Blocklist *head) { + return head->next == head; +} + +// list_is_singular - tests whether a list has just one entry. +// @head: the list to test. +template +bool ObjectPool::_blocklist_is_singular( + const Blocklist *head +) { + return !_blocklist_is_empty(head) && (head->next == head->prev); +} + +// Procedure: _for_each_block +template +template +void ObjectPool::_for_each_block(Blocklist* head, C&& c) { + Blocklist* p; + for(p=head->next; p!=head; p=p->next) { + c(_block_of(p)); + } +} + +// Procedure: _for_each_block_safe +// Iterate each item of a list - safe to free +template +template +void ObjectPool::_for_each_block_safe(Blocklist* head, C&& c) { + Blocklist* p; + Blocklist* t; + for(p=head->next, t=p->next; p!=head; p=t, t=p->next) { + c(_block_of(p)); + } +} + +// Function: _allocate +// allocate a spot from the block +template +T* ObjectPool::_allocate(Block* s) { + if(s->top == nullptr) { + return reinterpret_cast(s->data + s->i++ * X); + } + else { + T* retval = s->top; + s->top = *(reinterpret_cast(s->top)); + return retval; + } +} + +// Procedure: _deallocate +template +void ObjectPool::_deallocate(Block* s, T* ptr) { + *(reinterpret_cast(ptr)) = s->top; + s->top = ptr; +} + +// Function: allocate +template +template +T* ObjectPool::animate(ArgsT&&... args) { + + //std::cout << "construct a new item\n"; + + // my logically mapped heap + LocalHeap& h = _this_heap(); + + Block* s {nullptr}; + + h.mutex.lock(); + + // scan the list of superblocks from most full to least + int f = static_cast(F-1); + for(; f>=0; f--) { + if(!_blocklist_is_empty(&h.lists[f])) { + s = _block_of(h.lists[f].next); + break; + } + } + + // no superblock found + if(f == -1) { + + // check heap 0 for a superblock + _gheap.mutex.lock(); + if(!_blocklist_is_empty(&_gheap.list)) { + + s = _block_of(_gheap.list.next); + + //printf("get a superblock from global heap %lu\n", s->u); + assert(s->u < M && s->heap == nullptr); + f = static_cast(_bin(s->u + 1)); + + _blocklist_move_front(&s->list_node, &h.lists[f]); + + s->heap = &h; // must be within the global heap lock + _gheap.mutex.unlock(); + + h.u = h.u + s->u; + h.a = h.a + M; + } + // create a new block + else { + //printf("create a new superblock\n"); + _gheap.mutex.unlock(); + f = 0; + s = static_cast(std::malloc(sizeof(Block))); + + if(s == nullptr) { + throw std::bad_alloc(); + } + + s->heap = &h; + s->i = 0; + s->u = 0; + s->top = nullptr; + + _blocklist_push_front(&s->list_node, &h.lists[f]); + + h.a = h.a + M; + } + } + + // the superblock must have at least one space + //assert(s->u < M); + //printf("%lu %lu %lu\n", h.u, h.a, s->u); + //assert(h.u < h.a); + + h.u = h.u + 1; + s->u = s->u + 1; + + // take one item from the superblock + T* mem = _allocate(s); + + int b = static_cast(_bin(s->u)); + + if(b != f) { + //printf("move superblock from list[%d] to list[%d]\n", f, b); + _blocklist_move_front(&s->list_node, &h.lists[b]); + } + + //std::cout << "s.i " << s->i << '\n' + // << "s.u " << s->u << '\n' + // << "h.u " << h.u << '\n' + // << "h.a " << h.a << '\n'; + + h.mutex.unlock(); + + //printf("allocate %p (s=%p)\n", mem, s); + + new (mem) T(std::forward(args)...); + + mem->_object_pool_block = s; + + return mem; +} + +// Function: destruct +template +void ObjectPool::recycle(T* mem) { + + //Block* s = *reinterpret_cast( + // reinterpret_cast(mem) - sizeof(Block**) + //); + + //Block* s= *(reinterpret_cast(mem) - O); // (mem) - 1 + + Block* s = static_cast(mem->_object_pool_block); + + mem->~T(); + + //printf("deallocate %p (s=%p) M=%lu W=%lu X=%lu\n", mem, s, M, W, X); + + // here we need a loop because when we lock the heap, + // other threads may have removed the superblock to another heap + bool sync = false; + + do { + auto h = s->heap; + + // the block is in global heap + if(h == nullptr) { + std::lock_guard glock(_gheap.mutex); + if(s->heap == h) { + sync = true; + _deallocate(s, mem); + s->u = s->u - 1; + } + } + else { + std::lock_guard llock(h->mutex); + if(s->heap == h) { + sync = true; + // deallocate the item from the superblock + size_t f = _bin(s->u); + _deallocate(s, mem); + s->u = s->u - 1; + h->u = h->u - 1; + + size_t b = _bin(s->u); + + if(b != f) { + //printf("move superblock from list[%d] to list[%d]\n", f, b); + _blocklist_move_front(&s->list_node, &h->lists[b]); + } + + // transfer a mostly-empty superblock to global heap + if((h->u + K*M < h->a) && (h->u < ((F-1) * h->a / F))) { + for(size_t i=0; ilists[i])) { + Block* x = _block_of(h->lists[i].next); + //printf("transfer a block (x.u=%lu/x.i=%lu) to the global heap\n", x->u, x->i); + assert(h->u > x->u && h->a > M); + h->u = h->u - x->u; + h->a = h->a - M; + x->heap = nullptr; + std::lock_guard glock(_gheap.mutex); + _blocklist_move_front(&x->list_node, &_gheap.list); + break; + } + } + } + } + } + } while(!sync); + + //std::cout << "s.i " << s->i << '\n' + // << "s.u " << s->u << '\n'; +} + +// Function: _this_heap +template +typename ObjectPool::LocalHeap& +ObjectPool::_this_heap() { + // here we don't use thread local since object pool might be + // created and destroyed multiple times + thread_local auto hv = std::hash()(std::this_thread::get_id()); + return _lheaps[hv & _lheap_mask]; + + //return _lheaps[ + // std::hash()(std::this_thread::get_id()) & _lheap_mask + //]; +} + +// Function: _next_power_of_two +template +constexpr unsigned ObjectPool::_next_power_of_two(unsigned n) const { + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n++; + return n; +} + +} // end namespace tf -------------------------------------------------------- diff --git a/bundled/cpp-taskflow/include/taskflow/utility/os.hpp b/bundled/cpp-taskflow/include/taskflow/utility/os.hpp new file mode 100644 index 0000000000..c211d87ec3 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/utility/os.hpp @@ -0,0 +1,27 @@ +#include +#include +#include + +namespace tf { + +// Function: get_env +inline std::string get_env(const std::string& str) { +#ifdef _MSC_VER + char *ptr = nullptr; + size_t len = 0; + + if(_dupenv_s(&ptr, &len, str.c_str()) == 0 && ptr != nullptr) { + std::string res(ptr, len); + free(ptr); + return res; + } + return ""; + +#else + auto ptr = std::getenv(str.c_str()); + return ptr ? ptr : ""; +#endif +} + + +} // end of namespace tf ----------------------------------------------------- diff --git a/bundled/cpp-taskflow/include/taskflow/utility/passive_vector.hpp b/bundled/cpp-taskflow/include/taskflow/utility/passive_vector.hpp new file mode 100644 index 0000000000..b5710c054d --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/utility/passive_vector.hpp @@ -0,0 +1,213 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace tf { + +// Class: PassiveVector +// A vector storing only passive data structure (PDS) or POD data type. +template > +class PassiveVector { + + static_assert( + std::is_trivial::value && std::is_standard_layout::value, + "must be a plain old data type" + ); + + public: + + typedef T value_type; + typedef T & reference; + typedef const T & const_reference; + typedef T * pointer; + typedef const T * const_pointer; + typedef T * iterator; + typedef const T * const_iterator; + typedef std::reverse_iterator reverse_iterator; + typedef std::reverse_iterator const_reverse_iterator; + typedef ptrdiff_t difference_type; + typedef size_t size_type; + + PassiveVector() noexcept : + _data {reinterpret_cast(_stack)}, + _num {0}, + _cap {S} { + } + + explicit PassiveVector(size_type n) : _num {n} { + + // need to place on heap + if(n > S) { + _cap = n << 2; + _data = _allocator.allocate(_cap); + } + // stack + else { + _cap = S; + _data = reinterpret_cast(_stack); + } + + } + + PassiveVector(const PassiveVector& rhs) : _num {rhs._num} { + + // heap + if(rhs._num > S) { + _cap = rhs._cap; + _data = _allocator.allocate(rhs._cap); + } + else { + _cap = S; + _data = reinterpret_cast(_stack); + } + + std::memcpy(_data, rhs._data, _num * sizeof(T)); + } + + PassiveVector(PassiveVector&& rhs) : _num {rhs._num} { + + // rhs is in the stack + if(rhs.in_stack()) { + _cap = S; + _data = reinterpret_cast(_stack); + std::memcpy(_stack, rhs._stack, rhs._num*sizeof(T)); + } + // rhs is in the heap + else { + _cap = rhs._cap; + _data = rhs._data; + rhs._data = reinterpret_cast(rhs._stack); + rhs._cap = S; + } + + rhs._num = 0; + } + + ~PassiveVector() { + if(!in_stack()) { + _allocator.deallocate(_data, _cap); + } + } + + iterator begin() noexcept { return _data; } + const_iterator begin() const noexcept { return _data; } + const_iterator cbegin() const noexcept { return _data; } + iterator end() noexcept { return _data + _num; } + const_iterator end() const noexcept { return _data + _num; } + const_iterator cend() const noexcept { return _data + _num; } + + reverse_iterator rbegin() noexcept { return _data + _num; } + const_reverse_iterator crbegin() const noexcept { return _data + _num; } + reverse_iterator rend() noexcept { return _data; } + const_reverse_iterator crend() const noexcept { return _data; } + + reference operator [] (size_type idx) { return _data[idx]; } + const_reference operator [] (size_type idx) const { return _data[idx]; } + + reference at(size_type pos) { + if(pos >= _num) { + throw std::out_of_range("accessed position is out of range"); + } + return this->operator[](pos); + } + + const_reference at(size_type pos) const { + if(pos >= _num) { + throw std::out_of_range("accessed position is out of range"); + } + return this->operator[](pos); + } + + + reference front() { return _data[0]; } + const_reference front() const { return _data[0]; } + reference back() { return _data[_num-1]; } + const_reference back() const { return _data[_num-1]; } + + pointer data() noexcept { return _data; } + const_pointer data() const noexcept { return _data; } + + void push_back(const T& item) { + if(_num == _cap) { + _enlarge(_cap << 1); + } + _data[_num++] = item; + } + + void push_back(T&& item) { + if(_num == _cap) { + _enlarge(_cap << 1); + } + _data[_num++] = item; + } + + void pop_back() { + if(_num > 0) { + --_num; + } + } + + void clear() { + _num = 0; + } + + void resize(size_type N) { + if(N > _cap) { + _enlarge(N<<1); + } + _num = N; + } + + void reserve(size_type C) { + if(C > _cap) { + _enlarge(C); + } + } + + bool empty() const { return _num == 0; } + bool in_stack() const { return _data == reinterpret_cast(_stack); } + + size_type size() const { return _num; } + size_type capacity() const { return _cap; } + size_type max_size() const { return std::numeric_limits::max(); } + + bool operator == (const PassiveVector& rhs) const { + if(_num != rhs._num) { + return false; + } + return std::memcmp(_data, rhs._data, _num * sizeof(T)) == 0; + } + + private: + + char _stack[S*sizeof(T)]; + + T* _data; + + size_type _num; + size_type _cap; + + A _allocator; + + void _enlarge(size_type new_cap) { + + auto new_data = _allocator.allocate(new_cap); + + std::memcpy(new_data, _data, sizeof(T) * _num); + + if(!in_stack()) { + _allocator.deallocate(_data, _cap); + } + + _cap = new_cap; + _data = new_data; + } +}; + + +} // end of namespace tf. ---------------------------------------------------- + diff --git a/bundled/cpp-taskflow/include/taskflow/utility/singleton.hpp b/bundled/cpp-taskflow/include/taskflow/utility/singleton.hpp new file mode 100644 index 0000000000..de85e2848d --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/utility/singleton.hpp @@ -0,0 +1,33 @@ +#pragma once + +namespace tf { + +/** @class Singleton + +@brief class template to create a thread-safe singleton object + +*/ +template +class Singleton { + + public: + + /** + @brief get a reference to the singleton object + */ + static T& get() { + static T instance; + return instance; + } + + private: + + Singleton() = default; + ~Singleton() = default; + Singleton(const Singleton&)= delete; + Singleton& operator=(const Singleton&)= delete; +}; + + + +} // end of namespace tf ----------------------------------------------------- diff --git a/bundled/cpp-taskflow/include/taskflow/utility/stringify.hpp b/bundled/cpp-taskflow/include/taskflow/utility/stringify.hpp new file mode 100644 index 0000000000..6efd1a8c2b --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/utility/stringify.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +namespace tf { + +// Procedure: stringify +template +void ostreamize(std::ostringstream& oss, T&& token) { + oss << std::forward(token); +} + +// Procedure: stringify +template +void ostreamize(std::ostringstream& oss, T&& token, Rest&&... rest) { + oss << std::forward(token); + ostreamize(oss, std::forward(rest)...); +} + +} // end of namespace tf ----------------------------------------------------- diff --git a/bundled/cpp-taskflow/include/taskflow/utility/traits.hpp b/bundled/cpp-taskflow/include/taskflow/utility/traits.hpp new file mode 100644 index 0000000000..90080be1a3 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/utility/traits.hpp @@ -0,0 +1,307 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../nstd/variant.hpp" + +namespace tf { + +//----------------------------------------------------------------------------- +// Traits +//----------------------------------------------------------------------------- + +// Macro to check whether a class has a member function +#define define_has_member(member_name) \ +template \ +class has_member_##member_name \ +{ \ + typedef char yes_type; \ + typedef long no_type; \ + template static yes_type test(decltype(&U::member_name)); \ + template static no_type test(...); \ + public: \ + static constexpr bool value = sizeof(test(0)) == sizeof(yes_type); \ +} + +#define has_member(class_, member_name) has_member_##member_name::value + +// Struct: dependent_false +template +struct dependent_false { + static constexpr bool value = false; +}; + +template +constexpr auto dependent_false_v = dependent_false::value; + +//----------------------------------------------------------------------------- +// Move-On-Copy +//----------------------------------------------------------------------------- + +// Struct: MoC +template +struct MoC { + + MoC(T&& rhs) : object(std::move(rhs)) {} + MoC(const MoC& other) : object(std::move(other.object)) {} + + T& get() { return object; } + + mutable T object; +}; + +template +auto make_moc(T&& m) { + return MoC(std::forward(m)); +} + +//----------------------------------------------------------------------------- +// Functors. +//----------------------------------------------------------------------------- + +//// Overloadded. +//template +//struct Functors : Ts... { +// using Ts::operator()... ; +//}; +// +//template +//Functors(Ts...) -> Functors; + +// ---------------------------------------------------------------------------- +// callable traits +// ---------------------------------------------------------------------------- + +template +struct is_invocable : + std::is_constructible< + std::function, + std::reference_wrapper::type> + > { +}; + +template +constexpr bool is_invocable_v = is_invocable::value; + +template +struct is_invocable_r : + std::is_constructible< + std::function, + std::reference_wrapper::type> + > { +}; + +template +constexpr bool is_invocable_r_v = is_invocable_r::value; + + +// ---------------------------------------------------------------------------- +// Function Traits +// reference: https://github.com/ros2/rclcpp +// ---------------------------------------------------------------------------- + +template +struct tuple_tail; + +template +struct tuple_tail> { + using type = std::tuple; +}; + +// std::function +template +struct function_traits +{ + using arguments = typename tuple_tail< + typename function_traits::argument_tuple_type + >::type; + + static constexpr size_t arity = std::tuple_size::value; + + template + struct argument { + static_assert(N < arity, "error: invalid parameter index."); + using type = std::tuple_element_t; + }; + + template + using argument_t = typename argument::type; + + using return_type = typename function_traits::return_type; +}; + +// Free functions +template +struct function_traits { + + using return_type = R; + using argument_tuple_type = std::tuple; + + static constexpr size_t arity = sizeof...(Args); + + template + struct argument { + static_assert(N < arity, "error: invalid parameter index."); + using type = std::tuple_element_t>; + }; + + template + using argument_t = typename argument::type; +}; + +// function pointer +template +struct function_traits : function_traits { +}; + +// function reference +template +struct function_traits : function_traits { +}; + +// immutable lambda +template +struct function_traits + : function_traits +{}; + +// mutable lambda +template +struct function_traits + : function_traits +{}; + +/*// std::bind for object methods +template +#if defined _LIBCPP_VERSION // libc++ (Clang) +struct function_traits> +#elif defined _GLIBCXX_RELEASE // glibc++ (GNU C++ >= 7.1) +struct function_traits> +#elif defined __GLIBCXX__ // glibc++ (GNU C++) +struct function_traits(FArgs ...)>> +#elif defined _MSC_VER // MS Visual Studio +struct function_traits< + std::_Binder> +#else +#error "Unsupported C++ compiler / standard library" +#endif + : function_traits +{}; + +// std::bind for object const methods +template +#if defined _LIBCPP_VERSION // libc++ (Clang) +struct function_traits> +#elif defined _GLIBCXX_RELEASE // glibc++ (GNU C++ >= 7.1) +struct function_traits> +#elif defined __GLIBCXX__ // glibc++ (GNU C++) +struct function_traits(FArgs ...)>> +#elif defined _MSC_VER // MS Visual Studio +struct function_traits< + std::_Binder> +#else +#error "Unsupported C++ compiler / standard library" +#endif + : function_traits +{}; + +// std::bind for free functions +template +#if defined _LIBCPP_VERSION // libc++ (Clang) +struct function_traits> +#elif defined __GLIBCXX__ // glibc++ (GNU C++) +struct function_traits> +#elif defined _MSC_VER // MS Visual Studio +struct function_traits> +#else +#error "Unsupported C++ compiler / standard library" +#endif + : function_traits +{}; */ + +// decay to the raw type +template +struct function_traits : function_traits {}; + +template +struct function_traits : function_traits {}; + + +// ---------------------------------------------------------------------------- +// nstd::variant +// ---------------------------------------------------------------------------- +template +struct get_index; + +template +struct get_index_impl {}; + +template +struct get_index_impl : std::integral_constant{}; + +template +struct get_index_impl : get_index_impl{}; + +template +struct get_index> : get_index_impl<0, T, Ts...>{}; + +template +constexpr auto get_index_v = get_index::value; + +// ---------------------------------------------------------------------------- +// is_pod +//----------------------------------------------------------------------------- +template +struct is_pod { + static const bool value = std::is_trivial::value && + std::is_standard_layout::value; +}; + +template +constexpr bool is_pod_v = is_pod::value; + +// ---------------------------------------------------------------------------- +// bit_cast +//----------------------------------------------------------------------------- +template +typename std::enable_if< + (sizeof(To) == sizeof(From)) && + std::is_trivially_copyable::value && + std::is_trivial::value, + // this implementation requires that To is trivially default constructible + To +>::type +// constexpr support needs compiler magic +bit_cast(const From &src) noexcept { + To dst; + std::memcpy(&dst, &src, sizeof(To)); + return dst; +} + +} // end of namespace tf. --------------------------------------------------- + + + diff --git a/bundled/cpp-taskflow/include/taskflow/utility/uuid.hpp b/bundled/cpp-taskflow/include/taskflow/utility/uuid.hpp new file mode 100644 index 0000000000..d61afaa000 --- /dev/null +++ b/bundled/cpp-taskflow/include/taskflow/utility/uuid.hpp @@ -0,0 +1,236 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace tf { + +// Class: UUID +// +// A universally unique identifier (UUID) is an identifier standard used in software +// construction. A UUID is simply a 128-bit value. The meaning of each bit is defined +// by any of several variants. +// For human-readable display, many systems use a canonical format using hexadecimal +// text with inserted hyphen characters. +// +// For example: 123e4567-e89b-12d3-a456-426655440000 +// +// The intent of UUIDs is to enable distributed systems to uniquely identify information +// without significant central coordination. +// +// Copyright 2006 Andy Tompkins. +// Distributed under the Boost Software License, Version 1.0. (See +// accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// +struct UUID { + + using value_type = uint8_t; + using reference = uint8_t&; + using const_reference = const uint8_t&; + using iterator = uint8_t*; + using const_iterator = const uint8_t*; + using size_type = size_t; + using difference_type = ptrdiff_t; + + inline UUID(); + + UUID(const UUID&) = default; + UUID(UUID&&) = default; + + UUID& operator = (const UUID&) = default; + UUID& operator = (UUID&&) = default; + + inline static size_type size(); + inline iterator begin(); + inline const_iterator begin() const; + inline iterator end(); + inline const_iterator end() const; + + inline bool is_nil() const; + inline void swap(UUID& rhs); + inline size_t hash_value() const; + + inline bool operator == (const UUID&) const; + inline bool operator < (const UUID&) const; + inline bool operator > (const UUID&) const; + inline bool operator != (const UUID&) const; + inline bool operator >= (const UUID&) const; + inline bool operator <= (const UUID&) const; + + uint8_t data[16] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + inline std::string to_string() const; +}; + +// Constructor +inline UUID::UUID() { + + static thread_local std::random_device rd; + static thread_local std::mt19937 engine {rd()}; + + std::uniform_int_distribution distribution( + std::numeric_limits::min(), + std::numeric_limits::max() + ); + + int i = 0; + auto random_value = distribution(engine); + for (auto it=begin(); it!=end(); ++it, ++i) { + if (i == sizeof(unsigned long)) { + random_value = distribution(engine); + i = 0; + } + *it = static_cast((random_value >> (i*8)) & 0xFF); + } + + // set variant: must be 0b10xxxxxx + *(begin()+8) &= 0xBF; + *(begin()+8) |= 0x80; + + // set version: must be 0b0100xxxx + *(begin()+6) &= 0x4F; //0b01001111 + *(begin()+6) |= 0x40; //0b01000000 +} + +// Function: size +inline typename UUID::size_type UUID::size() { + return 16; +} + +// Function: begin +inline typename UUID::iterator UUID::begin() { + return data; +} + +// Function: begin +inline typename UUID::const_iterator UUID::begin() const { + return data; +} + +// Function: end +inline typename UUID::iterator UUID::end() { + return data+size(); +} + +// Function: end +inline typename UUID::const_iterator UUID::end() const { + return data+size(); +} + +// Function: is_nil +inline bool UUID::is_nil() const { + for (std::size_t i = 0; i < sizeof(this->data); ++i) { + if (this->data[i] != 0U) { + return false; + } + } + return true; +} + +// Procedure: swap +inline void UUID::swap(UUID& rhs) { + UUID tmp = *this; + *this = rhs; + rhs = tmp; +} + +// Function: hash_value +inline size_t UUID::hash_value() const { + size_t seed = 0; + for(auto i=begin(); i != end(); ++i) { + seed ^= static_cast(*i) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + } + return seed; +} + +// Operator: == +inline bool UUID::operator == (const UUID& rhs) const { + return std::memcmp(data, rhs.data, sizeof(data)) == 0; +} + +// Operator: != +inline bool UUID::operator != (const UUID& rhs) const { + return std::memcmp(data, rhs.data, sizeof(data)) != 0; +} + +// Operator: < +inline bool UUID::operator < (const UUID& rhs) const { + return std::memcmp(data, rhs.data, sizeof(data)) < 0; +} + +// Operator: > +inline bool UUID::operator > (const UUID& rhs) const { + return std::memcmp(data, rhs.data, sizeof(data)) > 0; +} + +// Operator: <= +inline bool UUID::operator <= (const UUID& rhs) const { + return std::memcmp(data, rhs.data, sizeof(data)) <= 0; +} + +// Operator: >= +inline bool UUID::operator >= (const UUID& rhs) const { + return std::memcmp(data, rhs.data, sizeof(data)) >= 0; +} + +// Function: to_string +inline std::string UUID::to_string() const { + + auto to_char = [](size_t i) { + if (i <= 9) return static_cast('0' + i); + return static_cast('a' + (i-10)); + }; + + std::string result; + result.reserve(36); + + std::size_t i=0; + for (auto it = begin(); it!=end(); ++it, ++i) { + + const size_t hi = ((*it) >> 4) & 0x0F; + result += to_char(hi); + + const size_t lo = (*it) & 0x0F; + result += to_char(lo); + + if (i == 3 || i == 5 || i == 7 || i == 9) { + result += '-'; + } + } + return result; +} + +// Procedure: swap +inline void swap(UUID& lhs, UUID& rhs) { + lhs.swap(rhs); +} + +// ostream +inline std::ostream& operator << (std::ostream& os, const UUID& rhs) { + os << rhs.to_string(); + return os; +} + +}; // End of namespace tf. --------------------------------------------------- + +//----------------------------------------------------------------------------- + + +namespace std { + +// Partial specialization: hash +template <> +struct hash { + size_t operator()(const tf::UUID& rhs) const { return rhs.hash_value(); } +}; + + +} // End of namespace std. --------------------------------------------------- + + diff --git a/bundled/setup_bundled.cmake b/bundled/setup_bundled.cmake index cb7a5cf3f9..0ad4f057b4 100644 --- a/bundled/setup_bundled.cmake +++ b/bundled/setup_bundled.cmake @@ -30,6 +30,18 @@ OPTION(DEAL_II_FORCE_BUNDLED_BOOST SET(BOOST_FOLDER "${CMAKE_SOURCE_DIR}/bundled/boost-1.70.0") +# +# CPP Taskflow +# + +SET(FEATURE_TASKFLOW_HAVE_BUNDLED TRUE) + +OPTION(DEAL_II_FORCE_BUNDLED_TASKFLOW + "Always use the bundled taskflow header library instead of an external one." + OFF) + +SET(TASKFLOW_FOLDER "${CMAKE_SOURCE_DIR}/bundled/cpp-taskflow") + # # Threading Building Blocks library @@ -40,9 +52,9 @@ IF( NOT CMAKE_SYSTEM_NAME MATCHES "CYGWIN" # # Cygwin is unsupported by tbb, Windows due to the way we compile tbb... # - SET(FEATURE_THREADS_HAVE_BUNDLED TRUE) + SET(FEATURE_TBB_HAVE_BUNDLED TRUE) - OPTION(DEAL_II_FORCE_BUNDLED_THREADS + OPTION(DEAL_II_FORCE_BUNDLED_TBB "Always use the bundled tbb library instead of an external one." OFF) diff --git a/cmake/configure/configure_1_taskflow.cmake b/cmake/configure/configure_1_taskflow.cmake new file mode 100644 index 0000000000..c16290664a --- /dev/null +++ b/cmake/configure/configure_1_taskflow.cmake @@ -0,0 +1,28 @@ +## --------------------------------------------------------------------- +## +## Copyright (C) 2012 - 2020 by the deal.II authors +## +## This file is part of the deal.II library. +## +## The deal.II library is free software; you can use it, redistribute +## it, and/or modify it under the terms of the GNU Lesser General +## Public License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## The full text of the license can be found in the file LICENSE.md at +## the top level directory of deal.II. +## +## --------------------------------------------------------------------- + +# +# Configuration for thread support in deal.II with the help of the CPP +# Taskflow library: +# + + +MACRO(FEATURE_CPP_TASKFLOW_CONFIGURE_BUNDLED) + LIST(APPEND CPP_TASKFLOW_BUNDLED_INCLUDE_DIRS ${CPP_TASKFLOW_FOLDER}/include) +ENDMACRO() + + +CONFIGURE_FEATURE(CPP_TASKFLOW) + diff --git a/cmake/configure/configure_1_tbb.cmake b/cmake/configure/configure_1_tbb.cmake index be585ac5dc..318e2f1145 100644 --- a/cmake/configure/configure_1_tbb.cmake +++ b/cmake/configure/configure_1_tbb.cmake @@ -63,10 +63,6 @@ MACRO(FEATURE_TBB_FIND_EXTERNAL var) ENDMACRO() -MACRO(FEATURE_TBB_CONFIGURE_EXTERNAL) -ENDMACRO() - - MACRO(FEATURE_TBB_CONFIGURE_BUNDLED) # # We have to disable a bunch of warnings: diff --git a/cmake/modules/FindCPP_TASKFLOW.cmake b/cmake/modules/FindCPP_TASKFLOW.cmake new file mode 100644 index 0000000000..791a914a5d --- /dev/null +++ b/cmake/modules/FindCPP_TASKFLOW.cmake @@ -0,0 +1,58 @@ +## --------------------------------------------------------------------- +## +## Copyright (C) 2020 by the deal.II authors +## +## This file is part of the deal.II library. +## +## The deal.II library is free software; you can use it, redistribute +## it, and/or modify it under the terms of the GNU Lesser General +## Public License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## The full text of the license can be found in the file LICENSE.md at +## the top level directory of deal.II. +## +## --------------------------------------------------------------------- + +# +# Try to find the Threading Building Blocks library +# +# This module exports +# +# CPP_TASKFLOW_INCLUDE_DIRS +# CPP_TASKFLOW_VERSION +# + +SET(CPP_TASKFLOW_DIR "" CACHE PATH "An optional hint to a CPP CPP_TASKFLOW installation") +SET_IF_EMPTY(CPP_TASKFLOW_DIR "$ENV{CPP_TASKFLOW_DIR}") + +FIND_PACKAGE(CPP_TASKFLOW_CONFIG + CONFIG QUIET + NAMES Cpp-Taskflow + HINTS + ${CPP_TASKFLOW_DIR}/lib/cmake/Cpp-Taskflow + ${CPP_TASKFLOW_DIR} + PATH_SUFFIXES + lib64/cmake/Cpp-Taskflow + lib/cmake/Cpp-Taskflow + lib${LIB_SUFFIX}/cmake/Cpp-Taskflow + NO_SYSTEM_ENVIRONMENT_PATH + ) + +SET(CPP_TASKFLOW_INCLUDE_DIR ${Cpp-Taskflow_INCLUDE_DIR}) + +# +# Extract version numbers: +# +SET(CPP_TASKFLOW_VERSION "${CPP_TASKFLOW_CONFIG_VERSION}") +STRING(REGEX REPLACE + "^([0-9]+).*$" "\\1" + CPP_TASKFLOW_VERSION_MAJOR "${CPP_TASKFLOW_CONFIG_VERSION}") +STRING(REGEX REPLACE + "^[0-9]+\\.([0-9]+).*$" "\\1" + CPP_TASKFLOW_VERSION_MINOR "${CPP_TASKFLOW_CONFIG_VERSION}") + +DEAL_II_PACKAGE_HANDLE(CPP_TASKFLOW + INCLUDE_DIRS REQUIRED CPP_TASKFLOW_INCLUDE_DIR + USER_INCLUDE_DIRS REQUIRED CPP_TASKFLOW_INCLUDE_DIR + CLEAR CPP_TASKFLOW_CONFIG_DIR + )