ENDIF()
-IF(FEATURE_THREADS_BUNDLED_CONFIGURED)
+IF(FEATURE_TASKFLOW_BUNDLED_CONFIGURED)
+ INSTALL(DIRECTORY ${TASKFLOW_FOLDER}/include/taskflow
+ DESTINATION ${DEAL_II_INCLUDE_RELDIR}/deal.II/bundled
+ COMPONENT library
+ )
+ENDIF()
+
+
+IF(FEATURE_TBB_BUNDLED_CONFIGURED)
INSTALL(DIRECTORY ${TBB_FOLDER}/include/tbb
DESTINATION ${DEAL_II_INCLUDE_RELDIR}/deal.II/bundled
COMPONENT library
A full version of the library can be downloaded at http://www.boost.org/.
+cpp-taskflow-*
+--------------
+
+Contains the cpp-taskflow project licensed under the MIT license.
+
+A full version of the FunctionParser project can be downloaded at
+https://cpp-taskflow.github.io/#/
+
+
muparser_*
----------
--- /dev/null
+MIT License
+
+Copyright (c) 2018-2020 T.-W. Huang, C.-X. Lin, G. Guo, and M. Wong
+
+University of Utah, Salt Lake City, UT, USA
+University of Illinois at Urbana-Champaign, IL, USA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- /dev/null
+# Cpp-Taskflow <img align="right" width="10%" src="image/cpp-taskflow_logo.png">
+
+[](https://app.codacy.com/app/tsung-wei-huang/cpp-taskflow?utm_source=github.com&utm_medium=referral&utm_content=cpp-taskflow/cpp-taskflow&utm_campaign=Badge_Grade_Dashboard)
+[](https://travis-ci.com/cpp-taskflow/cpp-taskflow)
+[](https://ci.appveyor.com/project/TsungWeiHuang/cpp-taskflow)
+[][wiki]
+[](https://cpp-taskflow.github.io/tfprof/)
+[](https://arxiv.org/abs/2004.10908v2)
+
+Cpp-Taskflow helps you quickly write parallel programs with high performance scalability
+and simultaneous high productivity.
+
+# Why Cpp-Taskflow?
+
+Cpp-Taskflow is faster, more expressive, and easier for drop-in integration
+than many of existing task programming frameworks
+in handling complex parallel workloads.
+
+
+
+Cpp-Taskflow lets you quickly implement task decomposition strategies
+that incorporate both regular and irregular compute patterns,
+together with an efficient *work-stealing* scheduler to optimize your multithreaded performance.
+
+| [Static Tasking](#get-started-with-cpp-taskflow) | [Dynamic Tasking](#dynamic-tasking) |
+| :------------: | :-------------: |
+|  | <img align="right" src="image/dynamic_graph.svg" width="100%"> |
+
+Cpp-Taskflow supports conditional tasking for you to make rapid control-flow decisions
+across dependent tasks to implement cycles and conditions that were otherwise difficult to do
+with existing tools.
+
+| [Conditional Tasking](#conditional-tasking) |
+| :-----------------: |
+|  |
+
+Cpp-Taskflow is composable. You can create large parallel graphs through
+composition of modular and reusable blocks that are easier to optimize
+at an individual scope.
+
+| [Taskflow Composition](#composable-tasking) |
+| :---------------: |
+||
+
+Cpp-Taskflow supports heterogeneous tasking for you to
+accelerate a wide range of scientific computing applications
+by harnessing the power of CPU-GPU collaborative computing.
+
+| [Concurrent CPU-GPU Tasking](#concurrent-cpu-gpu-tasking) |
+| :-----------------: |
+|  |
+
+We are committed to support trustworthy developments for both academic and industrial research projects
+in parallel computing. Check out [Who is Using Cpp-Taskflow](#who-is-using-cpp-taskflow) and what our users say:
+
++ *"Cpp-Taskflow is the cleanest Task API I've ever seen." [Damien Hocking @Corelium Inc](http://coreliuminc.com)*
++ *"Cpp-Taskflow has a very simple and elegant tasking interface. The performance also scales very well." [Glen Fraser][totalgee]*
++ *"Cpp-Taskflow lets me handle parallel processing in a smart way." [Hayabusa @Cpp-Learning](https://cpp-learning.com/cpp-taskflow/)*
++ *"Cpp-Taskflow improves the throughput of our graph engine in just a few hours of coding." [Jean-Michaël @KDAB](https://ossia.io/)*
++ *"Best poster award for open-source parallel programming library." [Cpp Conference 2018][Cpp Conference 2018]*
++ *"Second Prize of Open-source Software Competition." [ACM Multimedia Conference 2019](https://tsung-wei-huang.github.io/img/mm19-ossc-award.jpg)*
+
+See a quick [presentation][Presentation] and
+visit the [documentation][wiki] to learn more about Cpp-Taskflow.
+Technical details can be referred to our [arXiv paper](https://arxiv.org/abs/2004.10908v2).
+
+# Table of Contents
+
+* [Get Started with Cpp-Taskflow](#get-started-with-cpp-taskflow)
+* [Create a Taskflow Application](#create-a-taskflow-application)
+ * [Step 1: Create a Taskflow](#step-1-create-a-taskflow)
+ * [Step 2: Define Task Dependencies](#step-2-define-task-dependencies)
+ * [Step 3: Execute a Taskflow](#step-3-execute-a-taskflow)
+* [Dynamic Tasking](#dynamic-tasking)
+* [Conditional Tasking](#conditional-tasking)
+ * [Step 1: Create a Condition Task](#step-1-create-a-condition-task)
+ * [Step 2: Scheduling Rules for Condition Tasks](#step-2-scheduling-rules-for-condition-tasks)
+* [Composable Tasking](#composable-tasking)
+* [Concurrent CPU-GPU Tasking](#concurrent-cpu-gpu-tasking)
+ * [Step 1: Create a cudaFlow](#step-1-create-a-cudaflow)
+ * [Step 2: Compile and Execute a cudaFlow](#step-2-compile-and-execute-a-cudaflow)
+* [Visualize a Taskflow Graph](#visualize-a-taskflow-graph)
+* [Monitor Thread Activities](#monitor-thread-activities)
+* [API Reference](#api-reference)
+* [System Requirements](#system-requirements)
+* [Compile Unit Tests, Examples, and Benchmarks](#compile-unit-tests-examples-and-benchmarks)
+* [Who is Using Cpp-Taskflow?](#who-is-using-cpp-taskflow)
+
+
+# Get Started with Cpp-Taskflow
+
+The following example [simple.cpp](./examples/simple.cpp) shows the basic Cpp-Taskflow API
+you need in most applications.
+
+```cpp
+#include <taskflow/taskflow.hpp> // Cpp-Taskflow is header-only
+
+int main(){
+
+ tf::Executor executor;
+ tf::Taskflow taskflow;
+
+ auto [A, B, C, D] = taskflow.emplace(
+ [] () { std::cout << "TaskA\n"; }, // task dependency graph
+ [] () { std::cout << "TaskB\n"; }, //
+ [] () { std::cout << "TaskC\n"; }, // +---+
+ [] () { std::cout << "TaskD\n"; } // +---->| B |-----+
+ ); // | +---+ |
+ // +---+ +-v-+
+ A.precede(B); // A runs before B // | A | | D |
+ A.precede(C); // A runs before C // +---+ +-^-+
+ B.precede(D); // B runs before D // | +---+ |
+ C.precede(D); // C runs before D // +---->| C |-----+
+ // +---+
+ executor.run(taskflow).wait();
+
+ return 0;
+}
+```
+
+Compile and run the code with the following commands:
+
+```bash
+~$ g++ simple.cpp -I path/to/include/taskflow/ -std=c++17 -O2 -lpthread -o simple
+~$ ./simple
+TaskA
+TaskC <-- concurrent with TaskB
+TaskB <-- concurrent with TaskC
+TaskD
+```
+
+# Create a Taskflow Application
+
+Cpp-Taskflow defines a very expressive API to create task dependency graphs.
+Most applications are developed through the following three steps:
+
+## Step 1: Create a Taskflow
+
+Create a taskflow object to build a task dependency graph:
+
+```cpp
+tf::Taskflow taskflow;
+```
+
+A task is a callable object for which [std::invoke][std::invoke] is applicable.
+Use the method `emplace` to create a task:
+
+```cpp
+tf::Task A = taskflow.emplace([](){ std::cout << "Task A\n"; });
+```
+
+## Step 2: Define Task Dependencies
+
+You can add dependency links between tasks to enforce one task to run before or
+after another.
+
+```cpp
+A.precede(B); // A runs before B.
+```
+
+## Step 3: Execute a Taskflow
+
+To execute a taskflow, you need to create an *executor*.
+An executor manages a set of worker threads to execute a taskflow
+through an efficient *work-stealing* algorithm.
+
+```cpp
+tf::Executor executor;
+```
+
+The executor provides a rich set of methods to run a taskflow.
+You can run a taskflow multiple times, or until a stopping criteria is met.
+These methods are non-blocking with a [std::future][std::future] return
+to let you query the execution status.
+Executor is *thread-safe*.
+
+```cpp
+executor.run(taskflow); // runs the taskflow once
+executor.run_n(taskflow, 4); // runs the taskflow four times
+
+// keeps running the taskflow until the predicate becomes true
+executor.run_until(taskflow, [counter=4](){ return --counter == 0; } );
+```
+
+
+You can call `wait_for_all` to block the executor until all associated taskflows complete.
+
+```cpp
+executor.wait_for_all(); // block until all associated tasks finish
+```
+
+Notice that the executor does not own any taskflow.
+It is your responsibility to keep a taskflow alive during its execution,
+or it can result in undefined behavior.
+In most applications, you need only one executor to run multiple taskflows
+each representing a specific part of your parallel decomposition.
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# Dynamic Tasking
+
+Another powerful feature of Taskflow is *dynamic* tasking.
+Dynamic tasks are those tasks created during the execution of a taskflow.
+These tasks are spawned by a parent task and are grouped together to a *subflow* graph.
+To create a subflow for dynamic tasking,
+emplace a callable with one argument of type `tf::Subflow`.
+
+<img align="right" src="image/subflow_join.svg" width="30%">
+
+```cpp
+// create three regular tasks
+tf::Task A = tf.emplace([](){}).name("A");
+tf::Task C = tf.emplace([](){}).name("C");
+tf::Task D = tf.emplace([](){}).name("D");
+
+// create a subflow graph (dynamic tasking)
+tf::Task B = tf.emplace([] (tf::Subflow& subflow) {
+ tf::Task B1 = subflow.emplace([](){}).name("B1");
+ tf::Task B2 = subflow.emplace([](){}).name("B2");
+ tf::Task B3 = subflow.emplace([](){}).name("B3");
+ B1.precede(B3);
+ B2.precede(B3);
+}).name("B");
+
+A.precede(B); // B runs after A
+A.precede(C); // C runs after A
+B.precede(D); // D runs after B
+C.precede(D); // D runs after C
+```
+
+By default, a subflow graph joins its parent node.
+This ensures a subflow graph finishes before the successors of
+its parent task.
+You can disable this feature by calling `subflow.detach()`.
+For example, detaching the above subflow will result in the following execution flow:
+
+<img align="right" src="image/subflow_detach.svg" width="35%">
+
+```cpp
+// create a "detached" subflow graph (dynamic tasking)
+tf::Task B = tf.emplace([] (tf::Subflow& subflow) {
+ tf::Task B1 = subflow.emplace([](){}).name("B1");
+ tf::Task B2 = subflow.emplace([](){}).name("B2");
+ tf::Task B3 = subflow.emplace([](){}).name("B3");
+
+ B1.precede(B3);
+ B2.precede(B3);
+
+ // detach the subflow to form a parallel execution line
+ subflow.detach();
+}).name("B");
+```
+
+A subflow can be nested or recursive. You can create another subflow from
+the execution of a subflow and so on.
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# Conditional Tasking
+
+Taskflow supports *conditional tasking* for users to implement *dynamic* and *cyclic* control flows.
+You can create highly versatile and efficient parallel patterns through condition tasks.
+
+## Step 1: Create a Condition Task
+
+A *condition task* evalutes a set of instructions and returns an integer index
+of the next immediate successor to execute.
+The index is defined with respect to the order of its successor construction.
+
+<img align="right" src="image/condition-2.svg" width="20%">
+
+```cpp
+tf::Task init = tf.emplace([](){ }).name("init");
+tf::Task stop = tf.emplace([](){ }).name("stop");
+
+// creates a condition task that returns 0 or 1
+tf::Task cond = tf.emplace([](){
+ std::cout << "flipping a coin\n";
+ return rand() % 2;
+}).name("cond");
+
+// creates a feedback loop
+init.precede(cond);
+cond.precede(cond, stop); // cond--0-->cond, cond--1-->stop
+
+executor.run(tf).wait();
+```
+
+If the return value from `cond` is 0, it loops back to itself, or otherwise to `stop`.
+Cpp-Taskflow terms the preceding link from a condition task a *weak dependency*
+(dashed lines above).
+Others are *strong depedency* (solid lines above).
+
+
+## Step 2: Scheduling Rules for Condition Tasks
+
+When you submit a taskflow to an executor,
+the scheduler starts with tasks of *zero dependency* (both weak and strong dependencies)
+and continues to execute successive tasks whenever *strong dependencies* are met.
+However,
+the scheduler skips this rule for a condition task and jumps directly to its successor
+indexed by the return value.
+
+
+
+It is users' responsibility to ensure a taskflow is properly conditioned.
+Top things to avoid include no source tasks to start with and task race.
+The figure shows common pitfalls and their remedies.
+In the risky scenario, task X may not be raced if P and M is exclusively
+branching to X.
+
+
+
+
+A good practice for avoiding mistakes of conditional tasking is to infer the execution flow of your graphs based on our scheduling rules.
+Make sure there is no task race.
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+
+
+# Composable Tasking
+
+A powerful feature of `tf::Taskflow` is composability.
+You can create multiple task graphs from different parts of your workload
+and use them to compose a large graph through the `composed_of` method.
+
+
+<img align="right" src="image/composition.svg" width="50%">
+
+```cpp
+tf::Taskflow f1, f2;
+
+auto [f1A, f1B] = f1.emplace(
+ []() { std::cout << "Task f1A\n"; },
+ []() { std::cout << "Task f1B\n"; }
+);
+auto [f2A, f2B, f2C] = f2.emplace(
+ []() { std::cout << "Task f2A\n"; },
+ []() { std::cout << "Task f2B\n"; },
+ []() { std::cout << "Task f2C\n"; }
+);
+auto f1_module_task = f2.composed_of(f1);
+
+f1_module_task.succeed(f2A, f2B)
+ .precede(f2C);
+```
+
+Similarly, `composed_of` returns a task handle and you can use
+`precede` to create dependencies.
+You can compose a taskflow from multiple taskflows and use the result
+to compose a larger taskflow and so on.
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# Concurrent CPU-GPU Tasking
+
+Cpp-Taskflow enables concurrent CPU-GPU tasking by leveraging
+[Nvidia CUDA Toolkit][cuda-toolkit].
+You can harness the power of CPU-GPU collaborative computing
+to implement heterogeneous decomposition algorithms.
+
+## Step 1: Create a cudaFlow
+
+A `tf::cudaFlow` is a graph object created at runtime
+similar to dynamic tasking.
+It manages a task node in a taskflow and associates it
+with a [CUDA Graph][cudaGraph].
+To create a cudaFlow, emplace a callable with an argument
+of type `tf::cudaFlow`.
+
+
+
+```cpp
+tf::Taskflow taskflow;
+tf::Executor executor;
+
+const unsigned N = 1<<20; // size of the vector
+std::vector<float> hx(N, 1.0f), hy(N, 2.0f); // x and y vectors at host
+float *dx{nullptr}, *dy{nullptr}; // x and y vectors at device
+
+tf::Task allocate_x = taskflow.emplace([&](){ cudaMalloc(&dx, N*sizeof(float));});
+tf::Task allocate_y = taskflow.emplace([&](){ cudaMalloc(&dy, N*sizeof(float));});
+tf::Task cudaflow = taskflow.emplace([&](tf::cudaFlow& cf) {
+ tf::cudaTask h2d_x = cf.copy(dx, hx.data(), N); // host-to-device x data transfer
+ tf::cudaTask h2d_y = cf.copy(dy, hy.data(), N); // host-to-device y data transfer
+ tf::cudaTask d2h_x = cf.copy(hx.data(), dx, N); // device-to-host x data transfer
+ tf::cudaTask d2h_y = cf.copy(hy.data(), dy, N); // device-to-host y data transfer
+ // launch saxpy<<<(N+255)/256, 256, 0>>>(N, 2.0f, dx, dy)
+ tf::cudaTask kernel = cf.kernel((N+255)/256, 256, 0, saxpy, N, 2.0f, dx, dy);
+ kernel.succeed(h2d_x, h2d_y)
+ .precede(d2h_x, d2h_y);
+});
+cudaflow.succeed(allocate_x, allocate_y); // overlap data allocations
+
+executor.run(taskflow).wait();
+```
+
+Assume our kernel implements the canonical saxpy operation
+(single-precision A·X Plus Y) using the CUDA syntax.
+
+<img align="right" src="image/saxpy.svg" width="50%">
+
+```cpp
+// saxpy (single-precision A·X Plus Y) kernel
+__global__ void saxpy(
+ int n, float a, float *x, float *y
+) {
+ // get the thread index
+ int i = blockIdx.x*blockDim.x + threadIdx.x;
+
+ if (i < n) {
+ y[i] = a*x[i] + y[i];
+ }
+}
+```
+
+
+
+## Step 2: Compile and Execute a cudaFlow
+
+Name you source with the extension `.cu`, let's say `saxpy.cu`,
+and compile it through [nvcc][nvcc]:
+
+```bash
+~$ nvcc saxpy.cu -I path/to/include/taskflow -O2 -o saxpy
+~$ ./saxpy
+```
+
+Our source autonomously enables cudaFlow for compilers that support
+CUDA.
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# Visualize a Taskflow Graph
+
+You can dump a taskflow through a `std::ostream`
+in [GraphViz][GraphViz] format using the method `dump`.
+There are a number of free [GraphViz tools][AwesomeGraphViz] you could find online to visualize your Taskflow graph.
+
+<img align="right" src="image/graphviz.svg" width="25%">
+
+```cpp
+tf::Taskflow taskflow;
+tf::Task A = taskflow.emplace([] () {}).name("A");
+tf::Task B = taskflow.emplace([] () {}).name("B");
+tf::Task C = taskflow.emplace([] () {}).name("C");
+tf::Task D = taskflow.emplace([] () {}).name("D");
+tf::Task E = taskflow.emplace([] () {}).name("E");
+A.precede(B, C, E);
+C.precede(D);
+B.precede(D, E);
+
+taskflow.dump(std::cout); // dump the graph in DOT to std::cout
+```
+
+When you have tasks that are created at runtime (e.g., subflow, cudaFlow),
+you need to execute the graph first to spawn these tasks
+and dump the entire graph.
+
+<img align="right" src="image/debug_subflow.svg" width="25%">
+
+```cpp
+tf::Executor executor;
+tf::Taskflow taskflow;
+
+tf::Task A = taskflow.emplace([](){}).name("A");
+
+// create a subflow of two tasks B1->B2
+tf::Task B = taskflow.emplace([] (tf::Subflow& subflow) {
+ tf::Task B1 = subflow.emplace([](){}).name("B1");
+ tf::Task B2 = subflow.emplace([](){}).name("B2");
+ B1.precede(B2);
+}).name("B");
+
+A.precede(B);
+
+executor.run(tf).wait(); // run the taskflow to spawn subflows
+tf.dump(std::cout); // dump the graph including dynamic tasks
+```
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+
+
+
+# Monitor Thread Activities
+
+Cpp-Taskflow Profiler ([TFProf](https://github.com/cpp-taskflow/tfprof))
+provides the visualization and tooling needed for profiling cpp-taskflow programs.
+
+<p align="center">
+ <a href="https://cpp-taskflow.github.io/tfprof/">
+ <img width="100%" src="image/tfprof.png">
+ </a>
+</p>
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# API Reference
+
+The official [documentation][wiki] explains a complete list of
+Cpp-Taskflow API.
+Here, we highlight commonly used methods.
+
+## Taskflow API
+
+The class `tf::Taskflow` is the main place to create a task dependency graph.
+The table below summarizes a list of commonly used methods.
+
+| Method | Argument | Return | Description |
+| -------- | --------- | ------- | ----------- |
+| emplace | callables | tasks | creates a task with a given callable(s) |
+| placeholder | none | task | inserts a node without any work; work can be assigned later |
+| parallel_for | beg, end, callable, chunk | task pair | concurrently applies the callable chunk by chunk to the result of dereferencing every iterator in the range |
+| parallel_for | beg, end, step, callable, chunk | task pair | concurrently applies the callable chunk by chunk to an index-based range with a step size |
+| num_workers | none | size | queries the number of working threads in the pool |
+| dump | ostream | none | dumps the taskflow to an output stream in GraphViz format |
+
+### *emplace/placeholder*
+
+You can use `emplace` to create a task from a target callable.
+
+```cpp
+tf::Task task = tf.emplace([] () { std::cout << "my task\n"; });
+```
+
+When a task cannot be determined beforehand, you can create a placeholder and assign the callable later.
+
+```cpp
+tf::Task A = tf.emplace([](){});
+tf::Task B = tf.placeholder();
+A.precede(B);
+B.work([](){ /* do something */ });
+```
+
+### *parallel_for*
+
+The method `parallel_for` creates a subgraph that applies the callable to each item in the given range of a container.
+
+<img align="right" width="35%" src="image/parallel_for.svg">
+
+```cpp
+auto v = {'A', 'B', 'C', 'D'};
+auto [S, T] = tf.parallel_for(
+ v.begin(), // iterator to the beginning
+ v.end(), // iterator to the end
+ [] (int i) {
+ std::cout << "parallel " << i << '\n';
+ }
+);
+// add dependencies via S and T.
+```
+
+You can specify a *chunk* size (default one) in the last argument to force a task to include a certain number of items.
+
+<img align="right" width="18%" src="image/parallel_for_2.svg">
+
+```cpp
+auto v = {'A', 'B', 'C', 'D'};
+auto [S, T] = tf.parallel_for(
+ v.begin(), // iterator to the beginning
+ v.end(), // iterator to the end
+ [] (int i) {
+ std::cout << "AB and CD run in parallel" << '\n';
+ },
+ 2 // at least two items at a time
+);
+```
+
+In addition to iterator-based construction,
+`parallel_for` has another overload of index-based loop.
+The first three argument of this overload indicates
+starting index, ending index (exclusive), and step size.
+
+```cpp
+// [0, 11) with a step size of 2
+auto [S, T] = tf.parallel_for(
+ 0, 11, 2,
+ [] (int i) {
+ std::cout << "parallel_for on index " << i << std::endl;
+ },
+ 2 // at least two items at a time
+);
+// will print 0, 2, 4, 6, 8, 10 (three partitions, {0, 2}, {4, 6}, {8, 10})
+```
+
+## Task API
+
+Each time you create a task, the taskflow object adds a node to the present task dependency graph
+and return a *task handle* to you.
+A task handle is a lightweight object that defines a set of methods for users to
+access and modify the attributes of the associated task.
+The table below summarizes a list of commonly used methods.
+
+| Method | Argument | Return | Description |
+| -------------- | ----------- | ------ | ----------- |
+| name | string | self | assigns a human-readable name to the task |
+| work | callable | self | assigns a work of a callable object to the task |
+| precede | task list | self | enables this task to run *before* the given tasks |
+| succeed | task list | self | enables this task to run *after* the given tasks |
+| num_dependents | none | size | returns the number of dependents (inputs) of this task |
+| num_successors | none | size | returns the number of successors (outputs) of this task |
+| empty | none | bool | returns true if the task points to a graph node or false otherwise |
+| has_work | none | bool | returns true if the task points to a graph node with a callable assigned |
+
+### *name*
+
+The method `name` lets you assign a human-readable string to a task.
+
+```cpp
+A.name("my name is A");
+```
+
+### *work*
+
+The method `work` lets you assign a callable to a task.
+
+```cpp
+A.work([] () { std::cout << "hello world!"; });
+```
+
+### *precede*
+
+The method `precede` lets you add a preceding link from self to other tasks.
+
+<img align="right" width="30%" src="image/broadcast.svg">
+
+```cpp
+// A runs before B, C, D, and E
+A.precede(B, C, D, E);
+```
+
+The method `succeed` is similar to `precede` but operates in the opposite direction.
+
+### *empty/has_work*
+
+A task is empty is it is not associated with any graph node.
+
+```cpp
+tf::Task task; // assert(task.empty());
+```
+
+A placeholder task is associated with a graph node but has no work assigned yet.
+
+```
+tf::Task task = taskflow.placeholder(); // assert(!task.has_work());
+```
+
+## Executor API
+
+The class `tf::Executor` is used for execution of one or multiple taskflow objects.
+The table below summarizes a list of commonly used methods.
+
+| Method | Argument | Return | Description |
+| --------- | -------------- | ------------- | ------------------------ |
+| run | taskflow | future | runs the taskflow once |
+| run_n | taskflow, N | future | runs the taskflow N times |
+| run_until | taskflow, binary predicate | future | keeps running the taskflow until the predicate becomes true |
+| wait_for_all | none | none | blocks until all running tasks finish |
+| make_observer | arguments to forward to user-derived constructor | pointer to the observer | creates an observer to monitor the thread activities of the executor |
+
+### *run/run_n/run_until*
+
+The run series are non-blocking call to execute a taskflow graph.
+Issuing multiple runs on the same taskflow will automatically synchronize
+to a sequential chain of executions.
+
+```cpp
+executor.run(taskflow); // runs a graph once
+executor.run_n(taskflow, 5); // runs a graph five times
+executor.run_until(taskflow, my_pred); // keeps running until the my_pred becomes true
+executor.wait_for_all(); // blocks until all tasks finish
+```
+
+The first run finishes before the second run, and the second run finishes before the third run.
+ <div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# System Requirements
+
+To use the latest [Cpp-Taskflow](https://github.com/cpp-taskflow/cpp-taskflow/archive/master.zip), you only need a [C++14][C++14] compiler.
+
++ GNU C++ Compiler at least v5.0 with -std=c++14
++ Clang C++ Compiler at least v4.0 with -std=c++14
++ Microsoft Visual Studio at least v15.7 (MSVC++ 19.14); see [vcpkg guide](https://github.com/cpp-taskflow/cpp-taskflow/issues/143)
++ AppleClang Xode Version at least v8
++ Nvidia CUDA Toolkit and Compiler ([nvcc][nvcc]) at least v10.0 with -std=c++14
+
+Cpp-Taskflow works on Linux, Windows, and Mac OS X. See the [C++ compiler support](https://en.cppreference.com/w/cpp/compiler_support) status.
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# Compile Unit Tests, Examples, and Benchmarks
+
+Cpp-Taskflow uses [CMake](https://cmake.org/) to build examples and unit tests.
+We recommend using out-of-source build.
+
+```bash
+~$ cmake --version # must be at least 3.9 or higher
+~$ mkdir build
+~$ cd build
+~$ cmake ../
+~$ make & make test # run all unit tests
+```
+
+## Examples
+
+The folder `examples/` contains several examples and is a great place to learn to use Cpp-Taskflow.
+
+| Example | Description |
+| ------- | ----------- |
+| [simple.cpp](./examples/simple.cpp) | uses basic task building blocks to create a trivial taskflow graph |
+| [debug.cpp](./examples/debug.cpp)| inspects a taskflow through the dump method |
+| [parallel_for.cpp](./examples/parallel_for.cpp)| parallelizes a for loop with unbalanced workload |
+| [subflow.cpp](./examples/subflow.cpp)| demonstrates how to create a subflow graph that spawns three dynamic tasks |
+| [run_variants.cpp](./examples/run_variants.cpp)| shows multiple ways to run a taskflow graph |
+| [composition.cpp](./examples/composition.cpp)| demonstrates the decomposable interface of taskflow |
+| [observer.cpp](./examples/observer.cpp)| demonstrates how to monitor the thread activities in scheduling and running tasks |
+| [condition.cpp](./examples/condition.cpp) | creates a conditional tasking graph with a feedback loop control flow |
+| [cuda/saxpy.cu](./examples/cuda/saxpy.cu) | uses cudaFlow to create a saxpy (single-precision A·X Plus Y) task graph |
+| [cuda/matmul.cu](./examples/cuda/matmul.cu) | uses cudaFlow to create a matrix multiplication workload and compares it with a CPU basline |
+
+## Benchmarks
+
+Please visit [benchmarks](benchmarks/benchmarks.md) to learn to
+compile the benchmarks.
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# Who is Using Cpp-Taskflow?
+
+Cpp-Taskflow is being used in both industry and academic projects to scale up existing workloads
+that incorporate complex task dependencies.
+
+- [OpenTimer][OpenTimer]: A High-performance Timing Analysis Tool for Very Large Scale Integration (VLSI) Systems
+- [DtCraft][DtCraft]: A General-purpose Distributed Programming Systems using Data-parallel Streams
+- [Firestorm][Firestorm]: Fighting Game Engine with Asynchronous Resource Loaders (developed by [ForgeMistress][ForgeMistress])
+- [Shiva][Shiva]: An extensible engine via an entity component system through scripts, DLLs, and header-only (C++)
+- [PID Framework][PID Framework]: A Global Development Methodology Supported by a CMake API and Dedicated C++ Projects
+- [NovusCore][NovusCore]: An emulating project for World of Warraft (Wrath of the Lich King 3.3.5a 12340 client build)
+- [SA-PCB][SA-PCB]: Annealing-based Printed Circuit Board (PCB) Placement Tool
+- [LPMP](https://github.com/LPMP/LPMP): A C++ framework for developing scalable Lagrangian decomposition solvers for discrete optimization problems
+- [Heteroflow](https://github.com/Heteroflow/Heteroflow): A Modern C++ Parallel CPU-GPU Task Programming Library
+- [OpenPhySyn](https://github.com/The-OpenROAD-Project/OpenPhySyn): A plugin-based physical synthesis optimization kit as part of the OpenRoad flow
+- [OSSIA](https://ossia.io/): Open-source Software System for Interactive Applications
+
+[More...](https://github.com/search?q=cpp-taskflow&type=Code)
+
+<div align="right"><b><a href="#table-of-contents">[↑]</a></b></div>
+
+# Contributors
+
+Cpp-Taskflow is being actively developed and contributed by the
+[these people](https://github.com/cpp-taskflow/cpp-taskflow/graphs/contributors).
+Meanwhile, we appreciate the support from many organizations for our developments.
+
+
+| [<img src="image/utah.png" width="100px">][UofU] | [<img src="image/uiuc.png" width="100px">][UIUC] | [<img src="image/csl.png" width="100px">][CSL] | [<img src="image/nsf.png" width="100px">][NSF] | [<img src="image/darpa.png" width="100px">][DARPA IDEA] |
+| :---: | :---: | :---: | :---: | :---: |
+
+# License
+
+Cpp-Taskflow is licensed under the [MIT License](./LICENSE).
+
+* * *
+
+[Tsung-Wei Huang]: https://tsung-wei-huang.github.io/
+[Chun-Xun Lin]: https://github.com/clin99
+[Martin Wong]: https://ece.illinois.edu/directory/profile/mdfwong
+[Andreas Olofsson]: https://github.com/aolofsson
+[Gitter]: https://gitter.im/cpp-taskflow/Lobby
+[Gitter badge]: ./image/gitter_badge.svg
+[GitHub releases]: https://github.com/coo-taskflow/cpp-taskflow/releases
+[GitHub issues]: https://github.com/cpp-taskflow/cpp-taskflow/issues
+[GitHub insights]: https://github.com/cpp-taskflow/cpp-taskflow/pulse
+[GitHub pull requests]: https://github.com/cpp-taskflow/cpp-taskflow/pulls
+[GitHub contributors]: https://github.com/cpp-taskflow/cpp-taskflow/graphs/contributors
+[GraphViz]: https://www.graphviz.org/
+[AwesomeGraphViz]: https://dreampuf.github.io/GraphvizOnline/
+[OpenMP Tasking]: https://www.openmp.org/spec-html/5.0/openmpsu99.html
+[TBB FlowGraph]: https://www.threadingbuildingblocks.org/tutorial-intel-tbb-flow-graph
+[OpenTimer]: https://github.com/OpenTimer/OpenTimer
+[DtCraft]: https://github.com/tsung-wei-huang/DtCraft
+[totalgee]: https://github.com/totalgee
+[damienhocking]: https://github.com/damienhocking
+[ForgeMistress]: https://github.com/ForgeMistress
+[Patrik Huber]: https://github.com/patrikhuber
+[DARPA IDEA]: https://www.darpa.mil/news-events/2017-09-13
+[KingDuckZ]: https://github.com/KingDuckZ
+[NSF]: https://www.nsf.gov/
+[UIUC]: https://illinois.edu/
+[CSL]: https://csl.illinois.edu/
+[UofU]: https://www.utah.edu/
+[wiki]: https://cpp-taskflow.github.io/cpp-taskflow/index.html
+[release notes]: https://cpp-taskflow.github.io/cpp-taskflow/Releases.html
+[PayMe]: https://www.paypal.me/twhuang/10
+[C++17]: https://en.wikipedia.org/wiki/C%2B%2B17
+[C++14]: https://en.wikipedia.org/wiki/C%2B%2B14
+[email me]: mailto:twh760812@gmail.com
+[Cpp Conference 2018]: https://github.com/CppCon/CppCon2018
+[ChromeTracing]: https://www.chromium.org/developers/how-tos/trace-event-profiling-tool
+[IPDPS19]: https://tsung-wei-huang.github.io/papers/ipdps19.pdf
+[WorkStealing Wiki]: https://en.wikipedia.org/wiki/Work_stealing
+
+[std::invoke]: https://en.cppreference.com/w/cpp/utility/functional/invoke
+[std::future]: https://en.cppreference.com/w/cpp/thread/future
+
+[cuda-zone]: https://developer.nvidia.com/cuda-zone
+[nvcc]: https://developer.nvidia.com/cuda-llvm-compiler
+[cuda-toolkit]: https://developer.nvidia.com/cuda-toolkit
+[cudaGraph]: https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__GRAPH.html
+
+[Firestorm]: https://github.com/ForgeMistress/Firestorm
+[Shiva]: https://shiva.gitbook.io/project/shiva
+[PID Framework]: http://pid.lirmm.net/pid-framework/index.html
+[NovusCore]: https://github.com/novuscore/NovusCore
+[SA-PCB]: https://github.com/choltz95/SA-PCB
+
+[Presentation]: https://cpp-taskflow.github.io/
+[chrome://tracing]: chrome://tracing
+
--- /dev/null
+#pragma once
+
+#include <iostream>
+#include <sstream>
+#include <exception>
+
+#include "../utility/stringify.hpp"
+
+namespace tf {
+
+// Procedure: throw_se
+// Throws the system error under a given error code.
+template <typename... ArgsT>
+//void throw_se(const char* fname, const size_t line, Error::Code c, ArgsT&&... args) {
+void throw_re(const char* fname, const size_t line, ArgsT&&... args) {
+ std::ostringstream oss;
+ oss << "[" << fname << ":" << line << "] ";
+ ostreamize(oss, std::forward<ArgsT>(args)...);
+ //(oss << ... << args);
+ throw std::runtime_error(oss.str());
+}
+
+} // ------------------------------------------------------------------------
+
+#define TF_THROW(...) tf::throw_re(__FILE__, __LINE__, __VA_ARGS__);
+
--- /dev/null
+#pragma once
+
+#include "tsq.hpp"
+#include "notifier.hpp"
+#include "observer.hpp"
+#include "taskflow.hpp"
+
+namespace tf {
+
+
+/** @class WorkerView
+
+@brief class to access worker information from the observer interface
+
+*/
+//class WorkerView {
+//
+// friend class Executor;
+//
+// public:
+//
+//
+// private:
+//
+// Worker* _worker;
+//
+//};
+
+
+// ----------------------------------------------------------------------------
+// Executor Definition
+// ----------------------------------------------------------------------------
+
+
+/** @class Executor
+
+@brief execution interface for running a taskflow graph
+
+An executor object manages a set of worker threads and implements
+an efficient work-stealing scheduling algorithm to run a taskflow.
+
+*/
+class Executor {
+
+ struct Worker {
+ size_t id;
+ size_t victim;
+ Domain domain;
+ Executor* executor;
+ Notifier::Waiter* waiter;
+ std::mt19937 rdgen { std::random_device{}() };
+ TaskQueue<Node*> wsq[NUM_DOMAINS];
+ Node* cache {nullptr};
+ };
+
+ struct PerThread {
+ Worker* worker {nullptr};
+ };
+
+#ifdef TF_ENABLE_CUDA
+ struct cudaDevice {
+ std::vector<cudaStream_t> streams;
+ };
+#endif
+
+ public:
+
+#ifdef TF_ENABLE_CUDA
+ /**
+ @brief constructs the executor with N/M cpu/gpu worker threads
+ */
+ explicit Executor(
+ size_t N = std::thread::hardware_concurrency(),
+ size_t M = cuda_num_devices()
+ );
+#else
+ /**
+ @brief constructs the executor with N worker threads
+ */
+ explicit Executor(size_t N = std::thread::hardware_concurrency());
+#endif
+
+ /**
+ @brief destructs the executor
+ */
+ ~Executor();
+
+ /**
+ @brief runs the taskflow once
+
+ @param taskflow a tf::Taskflow object
+
+ @return a std::future to access the execution status of the taskflow
+ */
+ std::future<void> run(Taskflow& taskflow);
+
+ /**
+ @brief runs the taskflow once and invoke a callback upon completion
+
+ @param taskflow a tf::Taskflow object
+ @param callable a callable object to be invoked after this run
+
+ @return a std::future to access the execution status of the taskflow
+ */
+ template<typename C>
+ std::future<void> run(Taskflow& taskflow, C&& callable);
+
+ /**
+ @brief runs the taskflow for N times
+
+ @param taskflow a tf::Taskflow object
+ @param N number of runs
+
+ @return a std::future to access the execution status of the taskflow
+ */
+ std::future<void> run_n(Taskflow& taskflow, size_t N);
+
+ /**
+ @brief runs the taskflow for N times and then invokes a callback
+
+ @param taskflow a tf::Taskflow
+ @param N number of runs
+ @param callable a callable object to be invoked after this run
+
+ @return a std::future to access the execution status of the taskflow
+ */
+ template<typename C>
+ std::future<void> run_n(Taskflow& taskflow, size_t N, C&& callable);
+
+ /**
+ @brief runs the taskflow multiple times until the predicate becomes true and
+ then invokes a callback
+
+ @param taskflow a tf::Taskflow
+ @param pred a boolean predicate to return true for stop
+
+ @return a std::future to access the execution status of the taskflow
+ */
+ template<typename P>
+ std::future<void> run_until(Taskflow& taskflow, P&& pred);
+
+ /**
+ @brief runs the taskflow multiple times until the predicate becomes true and
+ then invokes the callback
+
+ @param taskflow a tf::Taskflow
+ @param pred a boolean predicate to return true for stop
+ @param callable a callable object to be invoked after this run
+
+ @return a std::future to access the execution status of the taskflow
+ */
+ template<typename P, typename C>
+ std::future<void> run_until(Taskflow& taskflow, P&& pred, C&& callable);
+
+ /**
+ @brief wait for all pending graphs to complete
+ */
+ void wait_for_all();
+
+ /**
+ @brief queries the number of worker threads (can be zero)
+ */
+ size_t num_workers() const;
+
+ /**
+ @brief queries the number of running topologies at the time of this call
+
+ When a taskflow is submitted to an executor, a topology is created to store
+ runtime metadata of the running taskflow.
+ */
+ size_t num_topologies() const;
+
+ /**
+ @brief queries the number of worker domains
+
+ Each domain manages a subset of worker threads to execute domain-specific tasks,
+ for example, HOST tasks and CUDA tasks.
+ */
+ size_t num_domains() const;
+
+ /**
+ @brief queries the id of the caller thread in this executor
+
+ Each worker has an unique id from 0 to N-1 exclusive to the associated executor.
+ If the caller thread does not belong to the executor, -1 is returned.
+ */
+ int this_worker_id() const;
+
+ /**
+ @brief constructs an observer to inspect the activities of worker threads
+
+ Each executor manage a list of observers in shared ownership with callers.
+
+ @tparam Observer observer type derived from tf::ObserverInterface
+ @tparam ArgsT... argument parameter pack
+
+ @param args arguments to forward to the constructor of the observer
+
+ @return a shared pointer to the created observer
+ */
+ template <typename Observer, typename... Args>
+ std::shared_ptr<Observer> make_observer(Args&&... args);
+
+ /**
+ @brief removes the associated observer
+ */
+ template <typename Observer>
+ void remove_observer(std::shared_ptr<Observer> observer);
+
+ /**
+ @brief queries the number of observers
+ */
+ size_t num_observers() const;
+
+ private:
+
+ const size_t _VICTIM_BEG;
+ const size_t _VICTIM_END;
+ const size_t _MAX_STEALS;
+ const size_t _MAX_YIELDS;
+
+ std::condition_variable _topology_cv;
+ std::mutex _topology_mutex;
+ std::mutex _wsq_mutex;
+
+ size_t _num_topologies {0};
+
+ std::vector<Worker> _workers;
+ std::vector<std::thread> _threads;
+
+#ifdef TF_ENABLE_CUDA
+ std::vector<cudaDevice> _cuda_devices;
+#endif
+
+ Notifier _notifier[NUM_DOMAINS];
+
+ TaskQueue<Node*> _wsq[NUM_DOMAINS];
+
+ size_t _id_offset[NUM_DOMAINS] = {0};
+
+ std::atomic<size_t> _num_actives[NUM_DOMAINS];
+ std::atomic<size_t> _num_thieves[NUM_DOMAINS];
+ std::atomic<bool> _done {0};
+
+ std::unordered_set<std::shared_ptr<ObserverInterface>> _observers;
+
+ TFProfObserver* _tfprof;
+
+ PerThread& _per_thread() const;
+
+ bool _wait_for_task(Worker&, Node*&);
+
+ void _instantiate_tfprof();
+ void _flush_tfprof();
+ void _observer_prologue(Worker&, Node*);
+ void _observer_epilogue(Worker&, Node*);
+ void _spawn(size_t, Domain);
+ void _worker_loop(Worker&);
+ void _exploit_task(Worker&, Node*&);
+ void _explore_task(Worker&, Node*&);
+ void _schedule(Node*, bool);
+ void _schedule(PassiveVector<Node*>&);
+ void _invoke(Worker&, Node*);
+ void _invoke_static_work(Worker&, Node*);
+ void _invoke_dynamic_work(Worker&, Node*, bool&);
+ void _invoke_condition_work(Worker&, Node*);
+ void _invoke_module_work(Worker&, Node*, bool&);
+
+#ifdef TF_ENABLE_CUDA
+ void _invoke_cudaflow_work(Worker&, Node*);
+ void _invoke_cudaflow_work_impl(Worker&, Node*);
+#endif
+
+ void _set_up_topology(Topology*);
+ void _tear_down_topology(Topology**);
+ void _increment_topology();
+ void _decrement_topology();
+ void _decrement_topology_and_notify();
+};
+
+
+#ifdef TF_ENABLE_CUDA
+// Constructor
+inline Executor::Executor(size_t N, size_t M) :
+ _VICTIM_BEG {0},
+ _VICTIM_END {N + M - 1},
+ _MAX_STEALS {(N + M + 1) << 1},
+ _MAX_YIELDS {100},
+ _workers {N + M},
+ _cuda_devices {cuda_num_devices()},
+ _notifier {Notifier(N), Notifier(M)} {
+
+ if(N == 0) {
+ TF_THROW("no cpu workers to execute taskflows");
+ }
+
+ if(M == 0) {
+ TF_THROW("no gpu workers to execute cudaflows");
+ }
+
+ for(int i=0; i<NUM_DOMAINS; ++i) {
+ _num_actives[i].store(0, std::memory_order_relaxed);
+ _num_thieves[i].store(0, std::memory_order_relaxed);
+ }
+
+ // create a per-worker stream on each cuda device
+ for(size_t i=0; i<_cuda_devices.size(); ++i) {
+ _cuda_devices[i].streams.resize(M);
+ cudaScopedDevice ctx(i);
+ for(size_t m=0; m<M; ++m) {
+ TF_CHECK_CUDA(
+ cudaStreamCreate(&(_cuda_devices[i].streams[m])),
+ "failed to create a cudaStream for worker ", m, " on device ", i
+ );
+ }
+ }
+
+ _spawn(N, HOST);
+ _spawn(M, CUDA);
+
+ // initiate the observer if requested
+ _instantiate_tfprof();
+}
+
+#else
+// Constructor
+inline Executor::Executor(size_t N) :
+ _VICTIM_BEG {0},
+ _VICTIM_END {N - 1},
+ _MAX_STEALS {(N + 1) << 1},
+ _MAX_YIELDS {100},
+ _workers {N},
+ _notifier {Notifier(N)} {
+
+ if(N == 0) {
+ TF_THROW("no cpu workers to execute taskflows");
+ }
+
+ for(int i=0; i<NUM_DOMAINS; ++i) {
+ _num_actives[i].store(0, std::memory_order_relaxed);
+ _num_thieves[i].store(0, std::memory_order_relaxed);
+ }
+
+ _spawn(N, HOST);
+
+ // instantite the default observer if requested
+ _instantiate_tfprof();
+}
+#endif
+
+// Destructor
+inline Executor::~Executor() {
+
+ // wait for all topologies to complete
+ wait_for_all();
+
+ // shut down the scheduler
+ _done = true;
+
+ for(int i=0; i<NUM_DOMAINS; ++i) {
+ _notifier[i].notify(true);
+ }
+
+ for(auto& t : _threads){
+ t.join();
+ }
+
+#ifdef TF_ENABLE_CUDA
+ // clean up the cuda streams
+ for(size_t i=0; i<_cuda_devices.size(); ++i) {
+ cudaScopedDevice ctx(i);
+ for(size_t m=0; m<_cuda_devices[i].streams.size(); ++m) {
+ cudaStreamDestroy(_cuda_devices[i].streams[m]);
+ }
+ }
+#endif
+
+ // flush the default observer
+ _flush_tfprof();
+}
+
+// Procedure: _instantiate_tfprof
+inline void Executor::_instantiate_tfprof() {
+ // TF_OBSERVER_TYPE
+ _tfprof = get_env("TF_ENABLE_PROFILER").empty() ?
+ nullptr : make_observer<TFProfObserver>().get();
+}
+
+// Procedure: _flush_tfprof
+inline void Executor::_flush_tfprof() {
+ if(_tfprof) {
+ std::ostringstream fpath;
+ fpath << get_env("TF_ENABLE_PROFILER") << _tfprof->_uuid << ".tfp";
+ std::ofstream ofs(fpath.str());
+ _tfprof->dump(ofs);
+ }
+}
+
+// Function: num_workers
+inline size_t Executor::num_workers() const {
+ return _workers.size();
+}
+
+// Function: num_domains
+inline size_t Executor::num_domains() const {
+ return NUM_DOMAINS;
+}
+
+// Function: num_topologies
+inline size_t Executor::num_topologies() const {
+ return _num_topologies;
+}
+
+// Function: _per_thread
+inline Executor::PerThread& Executor::_per_thread() const {
+ thread_local PerThread pt;
+ return pt;
+}
+
+// Function: this_worker_id
+inline int Executor::this_worker_id() const {
+ auto worker = _per_thread().worker;
+ return worker ? static_cast<int>(worker->id) : -1;
+}
+
+// Procedure: _spawn
+inline void Executor::_spawn(size_t N, Domain d) {
+
+ auto id = _threads.size();
+
+ _id_offset[d] = id;
+
+ for(size_t i=0; i<N; ++i, ++id) {
+
+ _workers[id].id = id;
+ _workers[id].victim = id;
+ _workers[id].domain = d;
+ _workers[id].executor = this;
+ _workers[id].waiter = &_notifier[d]._waiters[i];
+
+ _threads.emplace_back([this] (Worker& w) -> void {
+
+ PerThread& pt = _per_thread();
+ pt.worker = &w;
+
+ Node* t = nullptr;
+
+ // must use 1 as condition instead of !done
+ while(1) {
+
+ // execute the tasks.
+ _exploit_task(w, t);
+
+ // wait for tasks
+ if(_wait_for_task(w, t) == false) {
+ break;
+ }
+ }
+
+ }, std::ref(_workers[id]));
+ }
+
+}
+
+// Function: _explore_task
+inline void Executor::_explore_task(Worker& w, Node*& t) {
+
+ //assert(_workers[w].wsq.empty());
+ assert(!t);
+
+ const auto d = w.domain;
+
+ size_t num_steals = 0;
+ size_t num_yields = 0;
+
+ std::uniform_int_distribution<size_t> rdvtm(_VICTIM_BEG, _VICTIM_END);
+
+ //while(!_done) {
+ //
+ // size_t vtm = rdvtm(w.rdgen);
+ //
+ // t = (vtm == w.id) ? _wsq[d].steal() : _workers[vtm].wsq[d].steal();
+
+ // if(t) {
+ // break;
+ // }
+
+ // if(num_steal++ > _MAX_STEALS) {
+ // std::this_thread::yield();
+ // if(num_yields++ > _MAX_YIELDS) {
+ // break;
+ // }
+ // }
+ //}
+
+ do {
+ t = (w.id == w.victim) ? _wsq[d].steal() : _workers[w.victim].wsq[d].steal();
+
+ if(t) {
+ break;
+ }
+
+ if(num_steals++ > _MAX_STEALS) {
+ std::this_thread::yield();
+ if(num_yields++ > _MAX_YIELDS) {
+ break;
+ }
+ }
+
+ w.victim = rdvtm(w.rdgen);
+ } while(!_done);
+
+}
+
+// Procedure: _exploit_task
+inline void Executor::_exploit_task(Worker& w, Node*& t) {
+
+ assert(!w.cache);
+
+ if(t) {
+
+ const auto d = w.domain;
+
+ if(_num_actives[d].fetch_add(1) == 0 && _num_thieves[d] == 0) {
+ _notifier[d].notify(false);
+ }
+
+ auto tpg = t->_topology;
+ auto par = t->_parent;
+ auto exe = size_t{1};
+
+ do {
+ _invoke(w, t);
+
+ if(w.cache) {
+ t = w.cache;
+ w.cache = nullptr;
+ }
+ else {
+ t = w.wsq[d].pop();
+ if(t) {
+ // We only increment the counter when poping task from wsq
+ // (NOT including cache!)
+ if(t->_parent == par) {
+ exe++;
+ }
+ // joined subflow
+ else {
+ if(par == nullptr) {
+ // still have tasks so the topology join counter can't be zero
+ t->_topology->_join_counter.fetch_sub(exe);
+ }
+ else {
+ auto ret = par->_join_counter.fetch_sub(exe);
+ if(ret == exe) {
+ if(par->domain() == d) {
+ w.wsq[d].push(par);
+ }
+ else {
+ _schedule(par, false);
+ }
+ }
+ }
+ exe = 1;
+ par = t->_parent;
+ }
+ }
+ else {
+ // If no more local tasks!
+ if(par == nullptr) {
+ if(tpg->_join_counter.fetch_sub(exe) == exe) {
+ // TODO: Store tpg in local variable not in w
+ _tear_down_topology(&tpg);
+ if(tpg != nullptr) {
+ t = w.wsq[d].pop();
+ if(t) {
+ exe = 1;
+ }
+ }
+ }
+ }
+ else {
+ if(par->_join_counter.fetch_sub(exe) == exe) {
+ if(par->domain() == d) {
+ t = par;
+ par = par->_parent;
+ exe = 1;
+ }
+ else {
+ _schedule(par, false);
+ }
+ }
+ }
+ }
+ }
+ } while(t);
+
+ --_num_actives[d];
+ }
+}
+
+// Function: _wait_for_task
+inline bool Executor::_wait_for_task(Worker& worker, Node*& t) {
+
+ const auto d = worker.domain;
+
+ wait_for_task:
+
+ assert(!t);
+
+ ++_num_thieves[d];
+
+ explore_task:
+
+ _explore_task(worker, t);
+
+ if(t) {
+ if(_num_thieves[d].fetch_sub(1) == 1) {
+ _notifier[d].notify(false);
+ }
+ return true;
+ }
+
+ _notifier[d].prepare_wait(worker.waiter);
+
+ //if(auto vtm = _find_victim(me); vtm != _workers.size()) {
+ if(!_wsq[d].empty()) {
+
+ _notifier[d].cancel_wait(worker.waiter);
+ //t = (vtm == me) ? _wsq.steal() : _workers[vtm].wsq.steal();
+
+ t = _wsq[d].steal();
+ if(t) {
+ if(_num_thieves[d].fetch_sub(1) == 1) {
+ _notifier[d].notify(false);
+ }
+ return true;
+ }
+ else {
+ worker.victim = worker.id;
+ goto explore_task;
+ }
+ }
+
+ if(_done) {
+ _notifier[d].cancel_wait(worker.waiter);
+ for(int i=0; i<NUM_DOMAINS; ++i) {
+ _notifier[i].notify(true);
+ }
+ --_num_thieves[d];
+ return false;
+ }
+
+ if(_num_thieves[d].fetch_sub(1) == 1) {
+ if(_num_actives[d]) {
+ _notifier[d].cancel_wait(worker.waiter);
+ goto wait_for_task;
+ }
+ // check all domain queue again
+ for(auto& w : _workers) {
+ if(!w.wsq[d].empty()) {
+ worker.victim = w.id;
+ _notifier[d].cancel_wait(worker.waiter);
+ goto wait_for_task;
+ }
+ }
+ }
+
+ // Now I really need to relinguish my self to others
+ _notifier[d].commit_wait(worker.waiter);
+
+ return true;
+}
+
+// Function: make_observer
+template<typename Observer, typename... Args>
+std::shared_ptr<Observer> Executor::make_observer(Args&&... args) {
+
+ static_assert(
+ std::is_base_of<ObserverInterface, Observer>::value,
+ "Observer must be derived from ObserverInterface"
+ );
+
+ // use a local variable to mimic the constructor
+ auto ptr = std::make_shared<Observer>(std::forward<Args>(args)...);
+
+ ptr->set_up(_workers.size());
+
+ _observers.emplace(std::static_pointer_cast<ObserverInterface>(ptr));
+
+ return ptr;
+}
+
+// Procedure: remove_observer
+template <typename Observer>
+void Executor::remove_observer(std::shared_ptr<Observer> ptr) {
+
+ static_assert(
+ std::is_base_of<ObserverInterface, Observer>::value,
+ "Observer must be derived from ObserverInterface"
+ );
+
+ _observers.erase(std::static_pointer_cast<ObserverInterface>(ptr));
+}
+
+// Function: num_observers
+inline size_t Executor::num_observers() const {
+ return _observers.size();
+}
+
+// Procedure: _schedule
+// The main procedure to schedule a give task node.
+// Each task node has two types of tasks - regular and subflow.
+inline void Executor::_schedule(Node* node, bool bypass_hint) {
+
+ //assert(_workers.size() != 0);
+
+ const auto d = node->domain();
+
+ // caller is a worker to this pool
+ auto worker = _per_thread().worker;
+
+ if(worker != nullptr && worker->executor == this) {
+ if(bypass_hint) {
+ assert(!worker->cache);
+ worker->cache = node;
+ }
+ else {
+ worker->wsq[d].push(node);
+ if(worker->domain != d) {
+ if(_num_actives[d] == 0 && _num_thieves[d] == 0) {
+ _notifier[d].notify(false);
+ }
+ }
+ }
+ return;
+ }
+
+ // other threads
+ {
+ std::lock_guard<std::mutex> lock(_wsq_mutex);
+ _wsq[d].push(node);
+ }
+
+ _notifier[d].notify(false);
+}
+
+// Procedure: _schedule
+// The main procedure to schedule a set of task nodes.
+// Each task node has two types of tasks - regular and subflow.
+inline void Executor::_schedule(PassiveVector<Node*>& nodes) {
+
+ //assert(_workers.size() != 0);
+
+ // We need to cacth the node count to avoid accessing the nodes
+ // vector while the parent topology is removed!
+ const auto num_nodes = nodes.size();
+
+ if(num_nodes == 0) {
+ return;
+ }
+
+ // worker thread
+ auto worker = _per_thread().worker;
+
+ // task counts
+ size_t tcount[NUM_DOMAINS] = {0};
+
+ if(worker != nullptr && worker->executor == this) {
+ for(size_t i=0; i<num_nodes; ++i) {
+ const auto d = nodes[i]->domain();
+ worker->wsq[d].push(nodes[i]);
+ tcount[d]++;
+ }
+
+ for(int d=0; d<NUM_DOMAINS; ++d) {
+ if(tcount[d] && d != worker->domain) {
+ if(_num_actives[d] == 0 && _num_thieves[d] == 0) {
+ _notifier[d].notify_n(tcount[d]);
+ }
+ }
+ }
+
+ return;
+ }
+
+ // other threads
+ {
+ std::lock_guard<std::mutex> lock(_wsq_mutex);
+ for(size_t k=0; k<num_nodes; ++k) {
+ const auto d = nodes[k]->domain();
+ _wsq[d].push(nodes[k]);
+ tcount[d]++;
+ }
+ }
+
+ for(int d=0; d<NUM_DOMAINS; ++d) {
+ _notifier[d].notify_n(tcount[d]);
+ }
+}
+
+
+// Procedure: _invoke
+inline void Executor::_invoke(Worker& worker, Node* node) {
+
+ //assert(_workers.size() != 0);
+
+ // Here we need to fetch the num_successors first to avoid the invalid memory
+ // access caused by topology clear.
+ const auto num_successors = node->num_successors();
+
+ // acquire the parent flow counter
+ auto& c = (node->_parent) ? node->_parent->_join_counter :
+ node->_topology->_join_counter;
+
+ // switch is faster than nested if-else due to jump table
+ switch(node->_handle.index()) {
+ // static task
+ case Node::STATIC_WORK:{
+ _invoke_static_work(worker, node);
+ }
+ break;
+
+ // module task
+ case Node::MODULE_WORK: {
+ bool first_time = !node->_has_state(Node::SPAWNED);
+ bool emptiness = false;
+ _invoke_module_work(worker, node, emptiness);
+ if(first_time && !emptiness) {
+ return;
+ }
+ }
+ break;
+
+ // dynamic task
+ case Node::DYNAMIC_WORK: {
+ // Need to create a subflow if it is the first time entering here
+ if(!node->_has_state(Node::SPAWNED)) {
+ bool join = false;
+ _invoke_dynamic_work(worker, node, join);
+ if(join) {
+ return;
+ }
+ }
+ }
+ break;
+
+ // condition task
+ case Node::CONDITION_WORK: {
+ _invoke_condition_work(worker, node);
+ return ;
+ } // no need to add a break here due to the immediate return
+
+ // cudaflow task
+#ifdef TF_ENABLE_CUDA
+ case Node::CUDAFLOW_WORK: {
+ _invoke_cudaflow_work(worker, node);
+ }
+ break;
+#endif
+
+ // monostate
+ default:
+ break;
+ }
+
+
+ // We MUST recover the dependency since subflow may have
+ // a condition node to go back (cyclic).
+ // This must be done before scheduling the successors, otherwise this might cause
+ // race condition on the _dependents
+ if(node->_has_state(Node::BRANCH)) {
+ // If this is a case node, we need to deduct condition predecessors
+ node->_join_counter = node->num_strong_dependents();
+ }
+ else {
+ node->_join_counter = node->num_dependents();
+ }
+
+ node->_unset_state(Node::SPAWNED);
+
+ // At this point, the node storage might be destructed.
+ Node* cache {nullptr};
+
+ for(size_t i=0; i<num_successors; ++i) {
+ if(--(node->_successors[i]->_join_counter) == 0) {
+ if(node->_successors[i]->domain() != worker.domain) {
+ c.fetch_add(1);
+ _schedule(node->_successors[i], false);
+ }
+ else {
+ if(cache) {
+ c.fetch_add(1);
+ _schedule(cache, false);
+ }
+ cache = node->_successors[i];
+ }
+ }
+ }
+
+ if(cache) {
+ _schedule(cache, true);
+ }
+}
+
+// Procedure: _observer_prologue
+inline void Executor::_observer_prologue(Worker& worker, Node* node) {
+ for(auto& observer : _observers) {
+ observer->on_entry(worker.id, TaskView(node));
+ }
+}
+
+// Procedure: _observer_epilogue
+inline void Executor::_observer_epilogue(Worker& worker, Node* node) {
+ for(auto& observer : _observers) {
+ observer->on_exit(worker.id, TaskView(node));
+ }
+}
+
+// Procedure: _invoke_static_work
+inline void Executor::_invoke_static_work(Worker& worker, Node* node) {
+ _observer_prologue(worker, node);
+ nstd::get<Node::StaticWork>(node->_handle).work();
+ _observer_epilogue(worker, node);
+}
+
+// Procedure: _invoke_dynamic_work
+inline void Executor::_invoke_dynamic_work(Worker& worker, Node* node, bool& join) {
+
+ _observer_prologue(worker, node);
+
+ auto& subgraph = nstd::get<Node::DynamicWork>(node->_handle).subgraph;
+
+ subgraph.clear();
+ Subflow fb(subgraph);
+
+ nstd::get<Node::DynamicWork>(node->_handle).work(fb);
+
+ node->_set_state(Node::SPAWNED);
+
+ if(!subgraph.empty()) {
+
+ PassiveVector<Node*> src;
+
+ for(auto n : subgraph._nodes) {
+
+ n->_topology = node->_topology;
+ n->_set_up_join_counter();
+
+ if(!fb.detached()) {
+ n->_parent = node;
+ }
+
+ if(n->num_dependents() == 0) {
+ src.push_back(n);
+ }
+ }
+
+ join = fb.joined();
+
+ if(!join) { // Detach mode
+ node->_topology->_join_counter.fetch_add(src.size());
+ }
+ else { // Join mode (spawned nodes need second-round execution
+ node->_join_counter.fetch_add(src.size());
+
+ node->_parent ? node->_parent->_join_counter.fetch_add(1) :
+ node->_topology->_join_counter.fetch_add(1);
+ }
+
+ _schedule(src);
+ }
+
+ _observer_epilogue(worker, node);
+}
+
+// Procedure: _invoke_condition_work
+inline void Executor::_invoke_condition_work(Worker& worker, Node* node) {
+
+ _observer_prologue(worker, node);
+
+ if(node->_has_state(Node::BRANCH)) {
+ node->_join_counter = node->num_strong_dependents();
+ }
+ else {
+ node->_join_counter = node->num_dependents();
+ }
+
+ auto id = nstd::get<Node::ConditionWork>(node->_handle).work();
+
+ if(id >= 0 && static_cast<size_t>(id) < node->num_successors()) {
+ auto s = node->_successors[id];
+ s->_join_counter.store(0);
+
+ if(s->domain() == worker.domain) {
+ _schedule(s, true);
+ }
+ else {
+ node->_parent ? node->_parent->_join_counter.fetch_add(1) :
+ node->_topology->_join_counter.fetch_add(1);
+ _schedule(s, false);
+ }
+ }
+
+ _observer_epilogue(worker, node);
+}
+
+#ifdef TF_ENABLE_CUDA
+// Procedure: _invoke_cudaflow_work
+inline void Executor::_invoke_cudaflow_work(Worker& worker, Node* node) {
+ _observer_prologue(worker, node);
+ _invoke_cudaflow_work_impl(worker, node);
+ _observer_epilogue(worker, node);
+}
+
+// Procedure: _invoke_cudaflow_work_impl
+inline void Executor::_invoke_cudaflow_work_impl(Worker& w, Node* node) {
+
+ assert(w.domain == node->domain());
+
+ auto& h = nstd::get<Node::cudaFlowWork>(node->_handle);
+
+ h.graph.clear();
+
+ cudaFlow cf(h.graph, [repeat=1] () mutable { return repeat-- == 0; });
+
+ h.work(cf);
+
+ if(h.graph.empty()) {
+ return;
+ }
+
+ // transforms cudaFlow to a native cudaGraph under the specified device
+ // and launches the graph through a given or an internal device stream
+ const int d = cf._device;
+
+ cudaScopedDevice ctx(d);
+
+ auto s = cf._stream ? *(cf._stream) :
+ _cuda_devices[d].streams[w.id - _id_offset[w.domain]];
+
+ h.graph._make_native_graph();
+
+ cudaGraphExec_t exec;
+
+ TF_CHECK_CUDA(
+ cudaGraphInstantiate(&exec, h.graph._native_handle, nullptr, nullptr, 0),
+ "failed to create an executable cudaGraph"
+ );
+
+ while(!cf._predicate()) {
+ TF_CHECK_CUDA(
+ cudaGraphLaunch(exec, s), "failed to launch cudaGraph on stream ", s
+ );
+
+ TF_CHECK_CUDA(
+ cudaStreamSynchronize(s), "failed to synchronize stream ", s
+ );
+ }
+
+ TF_CHECK_CUDA(
+ cudaGraphExecDestroy(exec), "failed to destroy an executable cudaGraph"
+ );
+}
+#endif
+
+// Procedure: _invoke_module_work
+inline void Executor::_invoke_module_work(Worker& worker, Node* node, bool& ept) {
+
+ // second time to enter this context
+ if(node->_has_state(Node::SPAWNED)) {
+ return;
+ }
+
+ _observer_prologue(worker, node);
+
+ // first time to enter this context
+ node->_set_state(Node::SPAWNED);
+
+ auto module = nstd::get<Node::ModuleWork>(node->_handle).module;
+
+ if(module->empty()) {
+ ept = true;
+ return;
+ }
+
+ PassiveVector<Node*> src;
+
+ for(auto n: module->_graph._nodes) {
+
+ n->_topology = node->_topology;
+ n->_parent = node;
+ n->_set_up_join_counter();
+
+ if(n->num_dependents() == 0) {
+ src.push_back(n);
+ }
+ }
+
+ node->_join_counter.fetch_add(src.size());
+
+ if(node->_parent == nullptr) {
+ node->_topology->_join_counter.fetch_add(1);
+ }
+ else {
+ node->_parent->_join_counter.fetch_add(1);
+ }
+
+ // src can't be empty (banned outside)
+ _schedule(src);
+
+ _observer_epilogue(worker, node);
+}
+
+// Function: run
+inline std::future<void> Executor::run(Taskflow& f) {
+ return run_n(f, 1, [](){});
+}
+
+// Function: run
+template <typename C>
+std::future<void> Executor::run(Taskflow& f, C&& c) {
+ return run_n(f, 1, std::forward<C>(c));
+}
+
+// Function: run_n
+inline std::future<void> Executor::run_n(Taskflow& f, size_t repeat) {
+ return run_n(f, repeat, [](){});
+}
+
+// Function: run_n
+template <typename C>
+std::future<void> Executor::run_n(Taskflow& f, size_t repeat, C&& c) {
+ return run_until(f, [repeat]() mutable { return repeat-- == 0; }, std::forward<C>(c));
+}
+
+// Function: run_until
+template<typename P>
+std::future<void> Executor::run_until(Taskflow& f, P&& pred) {
+ return run_until(f, std::forward<P>(pred), [](){});
+}
+
+// Function: _set_up_topology
+inline void Executor::_set_up_topology(Topology* tpg) {
+
+ tpg->_sources.clear();
+
+ // scan each node in the graph and build up the links
+ for(auto node : tpg->_taskflow._graph._nodes) {
+
+ node->_topology = tpg;
+ node->_clear_state();
+
+ if(node->num_dependents() == 0) {
+ tpg->_sources.push_back(node);
+ }
+
+ node->_set_up_join_counter();
+ }
+
+ tpg->_join_counter.store(tpg->_sources.size(), std::memory_order_relaxed);
+}
+
+// Function: _tear_down_topology
+inline void Executor::_tear_down_topology(Topology** tpg) {
+
+ auto &f = (*tpg)->_taskflow;
+
+ //assert(&tpg == &(f._topologies.front()));
+
+ // case 1: we still need to run the topology again
+ if(! (*tpg)->_pred() ) {
+ //tpg->_recover_num_sinks();
+
+ assert((*tpg)->_join_counter == 0);
+ (*tpg)->_join_counter = (*tpg)->_sources.size();
+
+ _schedule((*tpg)->_sources);
+ }
+ // case 2: the final run of this topology
+ else {
+
+ if((*tpg)->_call != nullptr) {
+ (*tpg)->_call();
+ }
+
+ f._mtx.lock();
+
+ // If there is another run (interleave between lock)
+ if(f._topologies.size() > 1) {
+
+ assert((*tpg)->_join_counter == 0);
+
+ // Set the promise
+ (*tpg)->_promise.set_value();
+ f._topologies.pop_front();
+ f._mtx.unlock();
+
+ // decrement the topology but since this is not the last we don't notify
+ _decrement_topology();
+
+ *tpg = &(f._topologies.front());
+
+ _set_up_topology(*tpg);
+ _schedule((*tpg)->_sources);
+
+ //f._topologies.front()._bind(f._graph);
+ //*tpg = &(f._topologies.front());
+
+ //assert(f._topologies.front()._join_counter == 0);
+
+ //f._topologies.front()._join_counter = f._topologies.front()._sources.size();
+
+ //_schedule(f._topologies.front()._sources);
+ }
+ else {
+ assert(f._topologies.size() == 1);
+
+ // Need to back up the promise first here becuz taskflow might be
+ // destroy before taskflow leaves
+ auto p {std::move((*tpg)->_promise)};
+
+ f._topologies.pop_front();
+
+ f._mtx.unlock();
+
+ // We set the promise in the end in case taskflow leaves before taskflow
+ p.set_value();
+
+ _decrement_topology_and_notify();
+
+ // Reset topology so caller can stop execution
+ *tpg = nullptr;
+ }
+ }
+}
+
+// Function: run_until
+template <typename P, typename C>
+std::future<void> Executor::run_until(Taskflow& f, P&& pred, C&& c) {
+
+ _increment_topology();
+
+ // Special case of predicate
+ if(f.empty() || pred()) {
+ std::promise<void> promise;
+ promise.set_value();
+ _decrement_topology_and_notify();
+ return promise.get_future();
+ }
+
+ // Multi-threaded execution.
+ bool run_now {false};
+ Topology* tpg;
+ std::future<void> future;
+
+ {
+ std::lock_guard<std::mutex> lock(f._mtx);
+
+ // create a topology for this run
+ //tpg = &(f._topologies.emplace_back(f, std::forward<P>(pred), std::forward<C>(c)));
+ f._topologies.emplace_back(f, std::forward<P>(pred), std::forward<C>(c));
+ tpg = &(f._topologies.back());
+ future = tpg->_promise.get_future();
+
+ if(f._topologies.size() == 1) {
+ run_now = true;
+ //tpg->_bind(f._graph);
+ //_schedule(tpg->_sources);
+ }
+ }
+
+ // Notice here calling schedule may cause the topology to be removed sonner
+ // before the function leaves.
+ if(run_now) {
+ _set_up_topology(tpg);
+ _schedule(tpg->_sources);
+ }
+
+ return future;
+}
+
+// Procedure: _increment_topology
+inline void Executor::_increment_topology() {
+ std::lock_guard<std::mutex> lock(_topology_mutex);
+ ++_num_topologies;
+}
+
+// Procedure: _decrement_topology_and_notify
+inline void Executor::_decrement_topology_and_notify() {
+ std::lock_guard<std::mutex> lock(_topology_mutex);
+ if(--_num_topologies == 0) {
+ _topology_cv.notify_all();
+ }
+}
+
+// Procedure: _decrement_topology
+inline void Executor::_decrement_topology() {
+ std::lock_guard<std::mutex> lock(_topology_mutex);
+ --_num_topologies;
+}
+
+// Procedure: wait_for_all
+inline void Executor::wait_for_all() {
+ std::unique_lock<std::mutex> lock(_topology_mutex);
+ _topology_cv.wait(lock, [&](){ return _num_topologies == 0; });
+}
+
+} // end of namespace tf -----------------------------------------------------
+
+
--- /dev/null
+#pragma once
+
+#include "task.hpp"
+
+namespace tf {
+
+/**
+@class FlowBuilder
+
+@brief building methods of a task dependency graph
+
+*/
+class FlowBuilder {
+
+ friend class Task;
+
+ public:
+
+ /**
+ @brief constructs a flow builder object
+
+ @param graph a task dependency graph to manipulate
+ */
+ FlowBuilder(Graph& graph);
+
+ /**
+ @brief creates a static task from a given callable object
+
+ @tparam C callable type
+
+ @param callable a callable object constructible from std::function<void()>
+
+ @return Task handle
+ */
+ template <typename C>
+ std::enable_if_t<is_static_task_v<C>, Task> emplace(C&& callable);
+
+ /**
+ @brief creates a dynamic task from a given callable object
+
+ @tparam C callable type
+
+ @param callable a callable object constructible from std::function<void(Subflow&)>
+
+ @return Task handle
+ */
+ template <typename C>
+ std::enable_if_t<is_dynamic_task_v<C>, Task> emplace(C&& callable);
+
+ /**
+ @brief creates a condition task from a given callable object
+
+ @tparam C callable type
+
+ @param callable a callable object constructible from std::function<int()>
+
+ @return Task handle
+ */
+ template <typename C>
+ std::enable_if_t<is_condition_task_v<C>, Task> emplace(C&& callable);
+
+#ifdef TF_ENABLE_CUDA
+ /**
+ @brief creates a cudaflow task from a given callable object
+
+ @tparam C callable type
+
+ @param callable a callable object constructible from std::function<void(cudaFlow&)>
+
+ @return Task handle
+ */
+ template <typename C>
+ std::enable_if_t<is_cudaflow_task_v<C>, Task> emplace(C&& callable);
+#endif
+
+ /**
+ @brief creates multiple tasks from a list of callable objects
+
+ @tparam C... callable types
+
+ @param callables one or multiple callable objects constructible from each task category
+
+ @return a Task handle
+ */
+ template <typename... C, std::enable_if_t<(sizeof...(C)>1), void>* = nullptr>
+ auto emplace(C&&... callables);
+
+ /**
+ @brief creates a module task from a taskflow
+
+ @param taskflow a taskflow object for the module
+ @return a Task handle
+ */
+ Task composed_of(Taskflow& taskflow);
+
+ /**
+ @brief constructs a task dependency graph of range-based parallel_for
+
+ The task dependency graph applies a callable object
+ to the dereferencing of every iterator
+ in the range [beg, end) chunk by chunk.
+
+ @tparam I input iterator type
+ @tparam C callable type
+
+ @param beg iterator to the beginning (inclusive)
+ @param end iterator to the end (exclusive)
+ @param callable a callable object to be applied to
+ @param chunk size (default 1)
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <typename I, typename C>
+ std::pair<Task, Task> parallel_for(I beg, I end, C&& callable, size_t chunk=1);
+
+ /**
+ @brief constructs a task dependency graph of integer index-based parallel_for
+
+ The task dependency graph applies a callable object to every index
+ in the range [beg, end) with a step size chunk by chunk.
+
+ @tparam I integer (arithmetic) index type
+ @tparam C callable type
+
+ @param beg index of the beginning (inclusive)
+ @param end index of the end (exclusive)
+ @param step step size
+ @param callable a callable object to be applied to
+ @param chunk items per task
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <
+ typename I,
+ typename C,
+ std::enable_if_t<std::is_integral<std::decay_t<I>>::value, void>* = nullptr
+ >
+ std::pair<Task, Task> parallel_for(
+ I beg, I end, I step, C&& callable, size_t chunk = 1
+ );
+
+ /**
+ @brief constructs a task dependency graph of floating index-based parallel_for
+
+ The task dependency graph applies a callable object to every index
+ in the range [beg, end) with a step size chunk by chunk.
+
+ @tparam I floating (arithmetic) index type
+ @tparam C callable type
+
+ @param beg index of the beginning (inclusive)
+ @param end index of the end (exclusive)
+ @param step step size
+ @param callable a callable object to be applied to
+ @param chunk items per task
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <
+ typename I,
+ typename C,
+ std::enable_if_t<std::is_floating_point<std::decay_t<I>>::value, void>* = nullptr
+ >
+ std::pair<Task, Task> parallel_for(
+ I beg, I end, I step, C&& callable, size_t chunk = 1
+ );
+
+ /**
+ @brief construct a task dependency graph of parallel reduction
+
+ The task dependency graph reduces items in the range [beg, end) to a single result.
+
+ @tparam I input iterator type
+ @tparam T data type
+ @tparam B binary operator type
+
+ @param beg iterator to the beginning (inclusive)
+ @param end iterator to the end (exclusive)
+ @param result reference variable to store the final result
+ @param bop binary operator that will be applied in unspecified order to the result
+ of dereferencing the input iterator
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <typename I, typename T, typename B>
+ std::pair<Task, Task> reduce(I beg, I end, T& result, B&& bop);
+
+ /**
+ @brief constructs a task dependency graph of parallel reduction through @std_min
+
+ The task dependency graph applies a parallel reduction
+ to find the minimum item in the range [beg, end) through @std_min reduction.
+
+ @tparam I input iterator type
+ @tparam T data type
+
+ @param beg iterator to the beginning (inclusive)
+ @param end iterator to the end (exclusive)
+ @param result reference variable to store the final result
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <typename I, typename T>
+ std::pair<Task, Task> reduce_min(I beg, I end, T& result);
+
+ /**
+ @brief constructs a task dependency graph of parallel reduction through @std_max
+
+ The task dependency graph applies a parallel reduction
+ to find the maximum item in the range [beg, end) through @std_max reduction.
+
+ @tparam I input iterator type
+ @tparam T data type
+
+ @param beg iterator to the beginning (inclusive)
+ @param end iterator to the end (exclusive)
+ @param result reference variable to store the final result
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <typename I, typename T>
+ std::pair<Task, Task> reduce_max(I beg, I end, T& result);
+
+ /**
+ @brief constructs a task dependency graph of parallel transformation and reduction
+
+ The task dependency graph transforms each item in the range [beg, end)
+ into a new data type and then reduce the results.
+
+ @tparam I input iterator type
+ @tparam T data type
+ @tparam B binary operator
+ @tparam U unary operator type
+
+ @param beg iterator to the beginning (inclusive)
+ @param end iterator to the end (exclusive)
+ @param result reference variable to store the final result
+ @param bop binary function object that will be applied in unspecified order
+ to the results of @c uop; the return type must be @c T
+ @param uop unary function object that transforms each element
+ in the input range; the return type must be acceptable as input to @c bop
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <typename I, typename T, typename B, typename U>
+ std::pair<Task, Task> transform_reduce(I beg, I end, T& result, B&& bop, U&& uop);
+
+ /**
+ @brief constructs a task dependency graph of parallel transformation and reduction
+
+ The task dependency graph transforms each item in the range [beg, end)
+ into a new data type and then apply two-layer reductions to derive the result.
+
+ @tparam I input iterator type
+ @tparam T data type
+ @tparam B binary operator type
+ @tparam P binary operator type
+ @tparam U unary operator type
+
+ @param beg iterator to the beginning (inclusive)
+ @param end iterator to the end (exclusive)
+ @param result reference variable to store the final result
+ @param bop1 binary function object that will be applied in the second-layer reduction
+ to the results of @c bop2
+ @param bop2 binary function object that will be applied in the first-layer reduction
+ to the results of @c uop and the dereferencing of input iterators
+ @param uop unary function object that will be applied to transform an item to a new
+ data type that is acceptable as input to @c bop2
+
+ @return a pair of Task handles to the beginning and the end of the graph
+ */
+ template <typename I, typename T, typename B, typename P, typename U>
+ std::pair<Task, Task> transform_reduce(
+ I beg, I end, T& result, B&& bop1, P&& bop2, U&& uop
+ );
+
+ /**
+ @brief creates an empty task
+
+ @return a Task handle
+ */
+ Task placeholder();
+
+ /**
+ @brief adds a dependency link from task A to task B
+
+ @param A task A
+ @param B task B
+ */
+ void precede(Task A, Task B);
+
+ /**
+ @brief adds adjacent dependency links to a linear list of tasks
+
+ @param tasks a vector of tasks
+ */
+ void linearize(std::vector<Task>& tasks);
+
+ /**
+ @brief adds adjacent dependency links to a linear list of tasks
+
+ @param tasks an initializer list of tasks
+ */
+ void linearize(std::initializer_list<Task> tasks);
+
+ /**
+ @brief adds dependency links from one task A to many tasks
+
+ @param A task A
+ @param others a task set which A precedes
+ */
+ void broadcast(Task A, std::vector<Task>& others);
+
+ /**
+ @brief adds dependency links from one task A to many tasks
+
+ @param A task A
+ @param others a task set which A precedes
+ */
+ void broadcast(Task A, std::initializer_list<Task> others);
+
+ /**
+ @brief adds dependency links from many tasks to one task A
+
+ @param others a task set to precede A
+ @param A task A
+ */
+ void succeed(std::vector<Task>& others, Task A);
+
+ /**
+ @brief adds dependency links from many tasks to one task A
+
+ @param others a task set to precede A
+ @param A task A
+ */
+ void succeed(std::initializer_list<Task> others, Task A);
+
+ private:
+
+ Graph& _graph;
+
+ template <typename L>
+ void _linearize(L&);
+};
+
+// Constructor
+inline FlowBuilder::FlowBuilder(Graph& graph) :
+ _graph {graph} {
+}
+
+// Function: emplace
+template <typename... C, std::enable_if_t<(sizeof...(C)>1), void>*>
+auto FlowBuilder::emplace(C&&... cs) {
+ return std::make_tuple(emplace(std::forward<C>(cs))...);
+}
+
+// Function: emplace
+// emplaces a static task
+template <typename C>
+std::enable_if_t<is_static_task_v<C>, Task> FlowBuilder::emplace(C&& c) {
+ auto n = _graph.emplace_back(
+ nstd::in_place_type_t<Node::StaticWork>{}, std::forward<C>(c)
+ );
+ return Task(n);
+}
+
+// Function: emplace
+// emplaces a dynamic task
+template <typename C>
+std::enable_if_t<is_dynamic_task_v<C>, Task> FlowBuilder::emplace(C&& c) {
+ auto n = _graph.emplace_back(
+ nstd::in_place_type_t<Node::DynamicWork>{}, std::forward<C>(c)
+ );
+ return Task(n);
+}
+
+// Function: emplace
+// emplaces a condition task
+template <typename C>
+std::enable_if_t<is_condition_task_v<C>, Task> FlowBuilder::emplace(C&& c) {
+ auto n = _graph.emplace_back(
+ nstd::in_place_type_t<Node::ConditionWork>{}, std::forward<C>(c)
+ );
+ return Task(n);
+}
+
+#ifdef TF_ENABLE_CUDA
+// Function: emplace
+// emplaces a cudaflow task
+template <typename C>
+std::enable_if_t<is_cudaflow_task_v<C>, Task> FlowBuilder::emplace(C&& c) {
+ auto n = _graph.emplace_back(
+ nstd::in_place_type_t<Node::cudaFlowWork>{}, std::forward<C>(c)
+ );
+ return Task(n);
+}
+#endif
+
+// Function: composed_of
+inline Task FlowBuilder::composed_of(Taskflow& taskflow) {
+ auto node = _graph.emplace_back(
+ nstd::in_place_type_t<Node::ModuleWork>{}, &taskflow
+ );
+ return Task(node);
+}
+
+// Procedure: precede
+inline void FlowBuilder::precede(Task from, Task to) {
+ from._node->_precede(to._node);
+}
+
+// Procedure: broadcast
+inline void FlowBuilder::broadcast(Task from, std::vector<Task>& tos) {
+ for(auto to : tos) {
+ from.precede(to);
+ }
+}
+
+// Procedure: broadcast
+inline void FlowBuilder::broadcast(Task from, std::initializer_list<Task> tos) {
+ for(auto to : tos) {
+ from.precede(to);
+ }
+}
+
+// Function: succeed
+inline void FlowBuilder::succeed(std::vector<Task>& froms, Task to) {
+ for(auto from : froms) {
+ to.succeed(from);
+ }
+}
+
+// Function: succeed
+inline void FlowBuilder::succeed(std::initializer_list<Task> froms, Task to) {
+ for(auto from : froms) {
+ to.succeed(from);
+ }
+}
+
+// Function: placeholder
+inline Task FlowBuilder::placeholder() {
+ auto node = _graph.emplace_back();
+ return Task(node);
+}
+
+// Function: parallel_for
+template <typename I, typename C>
+std::pair<Task, Task> FlowBuilder::parallel_for(
+ I beg, I end, C&& c, size_t chunk
+){
+
+ //using category = typename std::iterator_traits<I>::iterator_category;
+
+ auto S = placeholder();
+ auto T = placeholder();
+
+ // default partition equals to the worker count
+ if(chunk == 0) {
+ chunk = 1;
+ }
+
+ size_t remain = std::distance(beg, end);
+
+ while(beg != end) {
+
+ auto e = beg;
+
+ auto x = std::min(remain, chunk);
+ std::advance(e, x);
+ remain -= x;
+
+ // Create a task
+ auto task = emplace([beg, e, c] () mutable {
+ std::for_each(beg, e, c);
+ });
+
+ S.precede(task);
+ task.precede(T);
+
+ // adjust the pointer
+ beg = e;
+ }
+
+ // special case
+ if(S.num_successors() == 0) {
+ S.precede(T);
+ }
+
+ return std::make_pair(S, T);
+}
+
+// Function: parallel_for
+template <
+ typename I,
+ typename C,
+ std::enable_if_t<std::is_integral<std::decay_t<I>>::value, void>*
+>
+std::pair<Task, Task> FlowBuilder::parallel_for(I beg, I end, I s, C&& c, size_t chunk) {
+
+ if((s == 0) || (beg < end && s <= 0) || (beg > end && s >=0) ) {
+ TF_THROW("invalid range [", beg, ", ", end, ") with step size ", s);
+ }
+
+ // source and target
+ auto source = placeholder();
+ auto target = placeholder();
+
+ if(chunk == 0) {
+ chunk = 1;
+ }
+
+ // positive case
+ if(beg < end) {
+ while(beg != end) {
+ auto o = static_cast<I>(chunk) * s;
+ auto e = std::min(beg + o, end);
+ auto task = emplace([=] () mutable {
+ for(auto i=beg; i<e; i+=s) {
+ c(i);
+ }
+ });
+ source.precede(task);
+ task.precede(target);
+ beg = e;
+ }
+ }
+ // negative case
+ else if(beg > end) {
+ while(beg != end) {
+ auto o = static_cast<I>(chunk) * s;
+ auto e = std::max(beg + o, end);
+ auto task = emplace([=] () mutable {
+ for(auto i=beg; i>e; i+=s) {
+ c(i);
+ }
+ });
+ source.precede(task);
+ task.precede(target);
+ beg = e;
+ }
+ }
+
+ if(source.num_successors() == 0) {
+ source.precede(target);
+ }
+
+ return std::make_pair(source, target);
+}
+
+// Function: parallel_for
+template <typename I, typename C,
+ std::enable_if_t<std::is_floating_point<std::decay_t<I>>::value, void>*
+>
+std::pair<Task, Task> FlowBuilder::parallel_for(I beg, I end, I s, C&& c, size_t chunk) {
+
+ if((s == 0) || (beg < end && s <= 0) || (beg > end && s >=0) ) {
+ TF_THROW("invalid range [", beg, ", ", end, ") with step size ", s);
+ }
+
+ // source and target
+ auto source = placeholder();
+ auto target = placeholder();
+
+ if(chunk == 0) {
+ chunk = 1;
+ }
+
+ // positive case
+ if(beg < end) {
+ size_t N=0;
+ I b = beg;
+ for(I e=beg; e<end; e+=s) {
+ if(++N == chunk) {
+ auto task = emplace([=] () mutable {
+ for(size_t i=0; i<N; ++i, b+=s) {
+ c(b);
+ }
+ });
+ source.precede(task);
+ task.precede(target);
+ N = 0;
+ b = e;
+ }
+ }
+
+ if(N) {
+ auto task = emplace([=] () mutable {
+ for(size_t i=0; i<N; ++i, b+=s) {
+ c(b);
+ }
+ });
+ source.precede(task);
+ task.precede(target);
+ }
+ }
+ else if(beg > end) {
+ size_t N=0;
+ I b = beg;
+ for(I e=beg; e>end; e+=s) {
+ if(++N == chunk) {
+ auto task = emplace([=] () mutable {
+ for(size_t i=0; i<N; ++i, b+=s) {
+ c(b);
+ }
+ });
+ source.precede(task);
+ task.precede(target);
+ N = 0;
+ b = e;
+ }
+ }
+
+ if(N) {
+ auto task = emplace([=] () mutable {
+ for(size_t i=0; i<N; ++i, b+=s) {
+ c(b);
+ }
+ });
+ source.precede(task);
+ task.precede(target);
+ }
+ }
+
+ if(source.num_successors() == 0) {
+ source.precede(target);
+ }
+
+ return std::make_pair(source, target);
+}
+
+// Function: reduce_min
+// Find the minimum element over a range of items.
+template <typename I, typename T>
+std::pair<Task, Task> FlowBuilder::reduce_min(I beg, I end, T& result) {
+ return reduce(beg, end, result, [] (const auto& l, const auto& r) {
+ return std::min(l, r);
+ });
+}
+
+// Function: reduce_max
+// Find the maximum element over a range of items.
+template <typename I, typename T>
+std::pair<Task, Task> FlowBuilder::reduce_max(I beg, I end, T& result) {
+ return reduce(beg, end, result, [] (const auto& l, const auto& r) {
+ return std::max(l, r);
+ });
+}
+
+// Function: transform_reduce
+template <typename I, typename T, typename B, typename U>
+std::pair<Task, Task> FlowBuilder::transform_reduce(
+ I beg, I end, T& result, B&& bop, U&& uop
+) {
+
+ //using category = typename std::iterator_traits<I>::iterator_category;
+
+ // Even partition
+ size_t d = std::distance(beg, end);
+ size_t w = std::max(unsigned{1}, std::thread::hardware_concurrency());
+ size_t g = std::max((d + w - 1) / w, size_t{2});
+
+ auto source = placeholder();
+ auto target = placeholder();
+
+ //std::vector<std::future<T>> futures;
+ auto g_results = std::make_unique<T[]>(w);
+ size_t id {0};
+
+ size_t remain = d;
+
+ while(beg != end) {
+
+ auto e = beg;
+
+ size_t x = std::min(remain, g);
+ std::advance(e, x);
+ remain -= x;
+
+ // Create a task
+ auto task = emplace([beg, e, bop, uop, res=&(g_results[id])] () mutable {
+ *res = uop(*beg);
+ for(++beg; beg != e; ++beg) {
+ *res = bop(std::move(*res), uop(*beg));
+ }
+ });
+
+ source.precede(task);
+ task.precede(target);
+
+ // adjust the pointer
+ beg = e;
+ id ++;
+ }
+
+ // target synchronizer
+ target.work([&result, bop, res=make_moc(std::move(g_results)), w=id] () {
+ for(auto i=0u; i<w; i++) {
+ result = bop(std::move(result), res.object[i]);
+ }
+ });
+
+ return std::make_pair(source, target);
+}
+
+// Function: transform_reduce
+template <typename I, typename T, typename B, typename P, typename U>
+std::pair<Task, Task> FlowBuilder::transform_reduce(
+ I beg, I end, T& result, B&& bop, P&& pop, U&& uop
+) {
+
+ //using category = typename std::iterator_traits<I>::iterator_category;
+
+ // Even partition
+ size_t d = std::distance(beg, end);
+ size_t w = std::max(unsigned{1}, std::thread::hardware_concurrency());
+ size_t g = std::max((d + w - 1) / w, size_t{2});
+
+ auto source = placeholder();
+ auto target = placeholder();
+
+ auto g_results = std::make_unique<T[]>(w);
+
+ size_t id {0};
+ size_t remain = d;
+
+ while(beg != end) {
+
+ auto e = beg;
+
+ size_t x = std::min(remain, g);
+ std::advance(e, x);
+ remain -= x;
+
+ // Create a task
+ auto task = emplace([beg, e, uop, pop, res= &g_results[id]] () mutable {
+ *res = uop(*beg);
+ for(++beg; beg != e; ++beg) {
+ *res = pop(std::move(*res), *beg);
+ }
+ });
+ source.precede(task);
+ task.precede(target);
+
+ // adjust the pointer
+ beg = e;
+ id ++;
+ }
+
+ // target synchronizer
+ target.work([&result, bop, g_results=make_moc(std::move(g_results)), w=id] () {
+ for(auto i=0u; i<w; i++) {
+ result = bop(std::move(result), std::move(g_results.object[i]));
+ }
+ });
+
+ return std::make_pair(source, target);
+}
+
+// Procedure: _linearize
+template <typename L>
+void FlowBuilder::_linearize(L& keys) {
+
+ auto itr = keys.begin();
+ auto end = keys.end();
+
+ if(itr == end) {
+ return;
+ }
+
+ auto nxt = itr;
+
+ for(++nxt; nxt != end; ++nxt, ++itr) {
+ itr->_node->_precede(nxt->_node);
+ }
+}
+
+// Procedure: linearize
+inline void FlowBuilder::linearize(std::vector<Task>& keys) {
+ _linearize(keys);
+}
+
+// Procedure: linearize
+inline void FlowBuilder::linearize(std::initializer_list<Task> keys) {
+ _linearize(keys);
+}
+
+// Proceduer: reduce
+template <typename I, typename T, typename B>
+std::pair<Task, Task> FlowBuilder::reduce(I beg, I end, T& result, B&& op) {
+
+ //using category = typename std::iterator_traits<I>::iterator_category;
+
+ size_t d = std::distance(beg, end);
+ size_t w = std::max(unsigned{1}, std::thread::hardware_concurrency());
+ size_t g = std::max((d + w - 1) / w, size_t{2});
+
+ auto source = placeholder();
+ auto target = placeholder();
+
+ //T* g_results = static_cast<T*>(malloc(sizeof(T)*w));
+ auto g_results = std::make_unique<T[]>(w);
+ //std::vector<std::future<T>> futures;
+
+ size_t id {0};
+ size_t remain = d;
+
+ while(beg != end) {
+
+ auto e = beg;
+
+ size_t x = std::min(remain, g);
+ std::advance(e, x);
+ remain -= x;
+
+ // Create a task
+ //auto [task, future] = emplace([beg, e, op] () mutable {
+ auto task = emplace([beg, e, op, res = &g_results[id]] () mutable {
+ *res = *beg;
+ for(++beg; beg != e; ++beg) {
+ *res = op(std::move(*res), *beg);
+ }
+ //auto init = *beg;
+ //for(++beg; beg != e; ++beg) {
+ // init = op(std::move(init), *beg);
+ //}
+ //return init;
+ });
+ source.precede(task);
+ task.precede(target);
+ //futures.push_back(std::move(future));
+
+ // adjust the pointer
+ beg = e;
+ id ++;
+ }
+
+ // target synchronizer
+ //target.work([&result, futures=MoC{std::move(futures)}, op] () {
+ // for(auto& fu : futures.object) {
+ // result = op(std::move(result), fu.get());
+ // }
+ //});
+ target.work([g_results=make_moc(std::move(g_results)), &result, op, w=id] () {
+ for(auto i=0u; i<w; i++) {
+ result = op(std::move(result), g_results.object[i]);
+ }
+ });
+
+ return std::make_pair(source, target);
+}
+
+// ----------------------------------------------------------------------------
+
+/**
+@class Subflow
+
+@brief building methods of a subflow graph in dynamic tasking
+
+*/
+class Subflow : public FlowBuilder {
+
+ public:
+
+ /**
+ @brief constructs a subflow builder object
+ */
+ template <typename... Args>
+ Subflow(Args&&... args);
+
+ /**
+ @brief enables the subflow to join its parent task
+ */
+ void join();
+
+ /**
+ @brief enables the subflow to detach from its parent task
+ */
+ void detach();
+
+ /**
+ @brief queries if the subflow will be detached from its parent task
+ */
+ bool detached() const;
+
+ /**
+ @brief queries if the subflow will join its parent task
+ */
+ bool joined() const;
+
+ private:
+
+ bool _detached {false};
+};
+
+// Constructor
+template <typename... Args>
+Subflow::Subflow(Args&&... args) :
+ FlowBuilder {std::forward<Args>(args)...} {
+}
+
+// Procedure: join
+inline void Subflow::join() {
+ _detached = false;
+}
+
+// Procedure: detach
+inline void Subflow::detach() {
+ _detached = true;
+}
+
+// Function: detached
+inline bool Subflow::detached() const {
+ return _detached;
+}
+
+// Function: joined
+inline bool Subflow::joined() const {
+ return !_detached;
+}
+
+
+// ----------------------------------------------------------------------------
+// Legacy code
+// ----------------------------------------------------------------------------
+
+using SubflowBuilder = Subflow;
+
+} // end of namespace tf. ---------------------------------------------------
+
+
--- /dev/null
+#pragma once
+
+#include "error.hpp"
+#include "../declarations.hpp"
+#include "../utility/object_pool.hpp"
+#include "../utility/traits.hpp"
+#include "../utility/passive_vector.hpp"
+#include "../utility/singleton.hpp"
+#include "../utility/uuid.hpp"
+#include "../utility/os.hpp"
+#include "../nstd/variant.hpp"
+
+#if defined(__CUDA__) || defined(__CUDACC__)
+#define TF_ENABLE_CUDA
+#include "../cuda/cuda_flow.hpp"
+#endif
+
+namespace tf {
+
+// ----------------------------------------------------------------------------
+// domain
+// ----------------------------------------------------------------------------
+
+enum Domain : int {
+ HOST = 0,
+#ifdef TF_ENABLE_CUDA
+ CUDA,
+#endif
+ NUM_DOMAINS
+};
+
+
+// ----------------------------------------------------------------------------
+// Class: Graph
+// ----------------------------------------------------------------------------
+class Graph {
+
+ friend class Node;
+ friend class Taskflow;
+ friend class Executor;
+
+ public:
+
+ Graph() = default;
+ Graph(const Graph&) = delete;
+ Graph(Graph&&);
+
+ ~Graph();
+
+ Graph& operator = (const Graph&) = delete;
+ Graph& operator = (Graph&&);
+
+ void clear();
+
+ bool empty() const;
+
+ size_t size() const;
+
+ template <typename ...Args>
+ Node* emplace_back(Args&& ...);
+
+ Node* emplace_back();
+
+ private:
+
+ static ObjectPool<Node>& _node_pool();
+
+ std::vector<Node*> _nodes;
+};
+
+// ----------------------------------------------------------------------------
+
+// Class: Node
+class Node {
+
+ friend class Task;
+ friend class TaskView;
+ friend class Topology;
+ friend class Taskflow;
+ friend class Executor;
+ friend class FlowBuilder;
+ friend class Subflow;
+
+ TF_ENABLE_POOLABLE_ON_THIS;
+
+ // state bit flag
+ constexpr static int SPAWNED = 0x1;
+ constexpr static int BRANCH = 0x2;
+
+ // static work handle
+ struct StaticWork {
+
+ template <typename C>
+ StaticWork(C&&);
+
+ std::function<void()> work;
+ };
+
+ // dynamic work handle
+ struct DynamicWork {
+
+ template <typename C>
+ DynamicWork(C&&);
+
+ std::function<void(Subflow&)> work;
+ Graph subgraph;
+ };
+
+ // condition work handle
+ struct ConditionWork {
+
+ template <typename C>
+ ConditionWork(C&&);
+
+ std::function<int()> work;
+ };
+
+ // module work handle
+ struct ModuleWork {
+
+ template <typename T>
+ ModuleWork(T&&);
+
+ Taskflow* module {nullptr};
+ };
+
+ // cudaFlow work handle
+#ifdef TF_ENABLE_CUDA
+ struct cudaFlowWork {
+
+ template <typename C>
+ cudaFlowWork(C&& c) : work {std::forward<C>(c)} {}
+
+ std::function<void(cudaFlow&)> work;
+
+ cudaGraph graph;
+ };
+#endif
+
+ using handle_t = nstd::variant<
+ nstd::monostate, // placeholder
+#ifdef TF_ENABLE_CUDA
+ cudaFlowWork, // cudaFlow
+#endif
+ StaticWork, // static tasking
+ DynamicWork, // dynamic tasking
+ ConditionWork, // conditional tasking
+ ModuleWork // composable tasking
+ >;
+
+ public:
+
+ // variant index
+ constexpr static auto PLACEHOLDER_WORK = get_index_v<nstd::monostate, handle_t>;
+ constexpr static auto STATIC_WORK = get_index_v<StaticWork, handle_t>;
+ constexpr static auto DYNAMIC_WORK = get_index_v<DynamicWork, handle_t>;
+ constexpr static auto CONDITION_WORK = get_index_v<ConditionWork, handle_t>;
+ constexpr static auto MODULE_WORK = get_index_v<ModuleWork, handle_t>;
+
+#ifdef TF_ENABLE_CUDA
+ constexpr static auto CUDAFLOW_WORK = get_index_v<cudaFlowWork, handle_t>;
+#endif
+
+ template <typename ...Args>
+ Node(Args&&... args);
+
+ ~Node();
+
+ size_t num_successors() const;
+ size_t num_dependents() const;
+ size_t num_strong_dependents() const;
+ size_t num_weak_dependents() const;
+
+ const std::string& name() const;
+
+ Domain domain() const;
+
+ private:
+
+ std::string _name;
+
+ handle_t _handle;
+
+ PassiveVector<Node*> _successors;
+ PassiveVector<Node*> _dependents;
+
+ Topology* _topology {nullptr};
+
+ Node* _parent {nullptr};
+
+ int _state {0};
+
+ std::atomic<size_t> _join_counter {0};
+
+ void _precede(Node*);
+ void _set_state(int);
+ void _unset_state(int);
+ void _clear_state();
+ void _set_up_join_counter();
+
+ bool _has_state(int) const;
+
+};
+
+// ----------------------------------------------------------------------------
+// Definition for Node::StaticWork
+// ----------------------------------------------------------------------------
+
+// Constructor
+template <typename C>
+Node::StaticWork::StaticWork(C&& c) : work {std::forward<C>(c)} {
+}
+
+// ----------------------------------------------------------------------------
+// Definition for Node::DynamicWork
+// ----------------------------------------------------------------------------
+
+// Constructor
+template <typename C>
+Node::DynamicWork::DynamicWork(C&& c) : work {std::forward<C>(c)} {
+}
+
+// ----------------------------------------------------------------------------
+// Definition for Node::ConditionWork
+// ----------------------------------------------------------------------------
+
+// Constructor
+template <typename C>
+Node::ConditionWork::ConditionWork(C&& c) : work {std::forward<C>(c)} {
+}
+
+// ----------------------------------------------------------------------------
+// Definition for Node::ModuleWork
+// ----------------------------------------------------------------------------
+
+// Constructor
+template <typename T>
+Node::ModuleWork::ModuleWork(T&& tf) : module {tf} {
+}
+
+// ----------------------------------------------------------------------------
+// Definition for Node
+// ----------------------------------------------------------------------------
+
+// Constructor
+template <typename ...Args>
+Node::Node(Args&&... args): _handle{std::forward<Args>(args)...} {
+}
+
+// Destructor
+inline Node::~Node() {
+ // this is to avoid stack overflow
+
+ if(_handle.index() == DYNAMIC_WORK) {
+
+ auto& subgraph = nstd::get<DynamicWork>(_handle).subgraph;
+
+ std::vector<Node*> nodes;
+
+ std::move(
+ subgraph._nodes.begin(), subgraph._nodes.end(), std::back_inserter(nodes)
+ );
+ subgraph._nodes.clear();
+
+ size_t i = 0;
+
+ while(i < nodes.size()) {
+
+ if(nodes[i]->_handle.index() == DYNAMIC_WORK) {
+
+ auto& sbg = nstd::get<DynamicWork>(nodes[i]->_handle).subgraph;
+ std::move(
+ sbg._nodes.begin(), sbg._nodes.end(), std::back_inserter(nodes)
+ );
+ sbg._nodes.clear();
+ }
+
+ ++i;
+ }
+
+ auto& np = Graph::_node_pool();
+ for(i=0; i<nodes.size(); ++i) {
+ //nodes[i]->~Node();
+ //np.deallocate(nodes[i]);
+ np.recycle(nodes[i]);
+ }
+ }
+}
+
+// Procedure: _precede
+inline void Node::_precede(Node* v) {
+ _successors.push_back(v);
+ v->_dependents.push_back(this);
+}
+
+// Function: num_successors
+inline size_t Node::num_successors() const {
+ return _successors.size();
+}
+
+// Function: dependents
+inline size_t Node::num_dependents() const {
+ return _dependents.size();
+}
+
+// Function: num_weak_dependents
+inline size_t Node::num_weak_dependents() const {
+ return std::count_if(
+ _dependents.begin(),
+ _dependents.end(),
+ [](Node* node){ return node->_handle.index() == Node::CONDITION_WORK; }
+ );
+}
+
+// Function: num_strong_dependents
+inline size_t Node::num_strong_dependents() const {
+ return std::count_if(
+ _dependents.begin(),
+ _dependents.end(),
+ [](Node* node){ return node->_handle.index() != Node::CONDITION_WORK; }
+ );
+}
+
+// Function: name
+inline const std::string& Node::name() const {
+ return _name;
+}
+
+// Function: domain
+inline Domain Node::domain() const {
+
+ Domain domain;
+
+ switch(_handle.index()) {
+
+ case STATIC_WORK:
+ case DYNAMIC_WORK:
+ case CONDITION_WORK:
+ case MODULE_WORK:
+ domain = Domain::HOST;
+ break;
+
+#ifdef TF_ENABLE_CUDA
+ case CUDAFLOW_WORK:
+ domain = Domain::CUDA;
+ break;
+#endif
+
+ default:
+ domain = Domain::HOST;
+ break;
+ }
+
+ return domain;
+}
+
+//
+//// Function: dump
+//inline std::string Node::dump() const {
+// std::ostringstream os;
+// dump(os);
+// return os.str();
+//}
+//
+//// Function: dump
+//inline void Node::dump(std::ostream& os) const {
+//
+// os << 'p' << this << "[label=\"";
+// if(_name.empty()) os << 'p' << this;
+// else os << _name;
+// os << "\" ";
+//
+// // condition node is colored green
+// if(_handle.index() == CONDITION_WORK) {
+// os << " shape=diamond color=black fillcolor=aquamarine style=filled";
+// }
+//
+// os << "];\n";
+//
+// for(size_t s=0; s<_successors.size(); ++s) {
+// if(_handle.index() == CONDITION_WORK) {
+// // case edge is dashed
+// os << 'p' << this << " -> p" << _successors[s]
+// << " [style=dashed label=\"" << s << "\"];\n";
+// }
+// else {
+// os << 'p' << this << " -> p" << _successors[s] << ";\n";
+// }
+// }
+//
+// // subflow join node
+// if(_parent && _successors.size() == 0) {
+// os << 'p' << this << " -> p" << _parent << ";\n";
+// }
+//
+// if(_subgraph && !_subgraph->empty()) {
+//
+// os << "subgraph cluster_p" << this << " {\nlabel=\"Subflow: ";
+// if(_name.empty()) os << 'p' << this;
+// else os << _name;
+//
+// os << "\";\n" << "color=blue\n";
+//
+// for(const auto& n : _subgraph->nodes()) {
+// n->dump(os);
+// }
+// os << "}\n";
+// }
+//}
+
+// Procedure: _set_state
+inline void Node::_set_state(int flag) {
+ _state |= flag;
+}
+
+// Procedure: _unset_state
+inline void Node::_unset_state(int flag) {
+ _state &= ~flag;
+}
+
+// Procedure: _clear_state
+inline void Node::_clear_state() {
+ _state = 0;
+}
+
+// Procedure: _set_up_join_counter
+inline void Node::_set_up_join_counter() {
+
+ int c = 0;
+
+ for(auto p : _dependents) {
+ if(p->_handle.index() == Node::CONDITION_WORK) {
+ _set_state(Node::BRANCH);
+ }
+ else {
+ c++;
+ }
+ }
+
+ _join_counter.store(c, std::memory_order_relaxed);
+}
+
+// Function: _has_state
+inline bool Node::_has_state(int flag) const {
+ return _state & flag;
+}
+
+// ----------------------------------------------------------------------------
+// Graph definition
+// ----------------------------------------------------------------------------
+
+// Function: _node_pool
+inline ObjectPool<Node>& Graph::_node_pool() {
+ static ObjectPool<Node> pool;
+ return pool;
+}
+
+// Destructor
+inline Graph::~Graph() {
+ auto& np = _node_pool();
+ for(auto node : _nodes) {
+ //node->~Node();
+ //np.deallocate(node);
+ np.recycle(node);
+ }
+}
+
+// Move constructor
+inline Graph::Graph(Graph&& other) :
+ _nodes {std::move(other._nodes)} {
+}
+
+// Move assignment
+inline Graph& Graph::operator = (Graph&& other) {
+ _nodes = std::move(other._nodes);
+ return *this;
+}
+
+// Procedure: clear
+inline void Graph::clear() {
+ auto& np = _node_pool();
+ for(auto node : _nodes) {
+ //node->~Node();
+ //np.deallocate(node);
+ np.recycle(node);
+ }
+ _nodes.clear();
+}
+
+// Function: size
+// query the size
+inline size_t Graph::size() const {
+ return _nodes.size();
+}
+
+// Function: empty
+// query the emptiness
+inline bool Graph::empty() const {
+ return _nodes.empty();
+}
+
+// Function: emplace_back
+// create a node from a give argument; constructor is called if necessary
+template <typename ...ArgsT>
+Node* Graph::emplace_back(ArgsT&&... args) {
+ //auto node = _node_pool().allocate();
+ //new (node) Node(std::forward<ArgsT>(args)...);
+ //_nodes.push_back(node);
+ _nodes.push_back(_node_pool().animate(std::forward<ArgsT>(args)...));
+ return _nodes.back();
+}
+
+// Function: emplace_back
+// create a node from a give argument; constructor is called if necessary
+inline Node* Graph::emplace_back() {
+ //auto node = _node_pool().allocate();
+ //new (node) Node();
+ //_nodes.push_back(node);
+ _nodes.push_back(_node_pool().animate());
+ return _nodes.back();
+}
+
+
+} // end of namespace tf. ---------------------------------------------------
+
+
+
+
+
--- /dev/null
+// 2019/02/09 - created by Tsung-Wei Huang
+// - modified the event count from Eigen
+
+#pragma once
+
+#include <iostream>
+#include <vector>
+#include <cstdlib>
+#include <cstdio>
+#include <atomic>
+#include <memory>
+#include <deque>
+#include <mutex>
+#include <condition_variable>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+#include <cassert>
+
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2016 Dmitry Vyukov <dvyukov@google.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+namespace tf {
+
+// Notifier allows to wait for arbitrary predicates in non-blocking
+// algorithms. Think of condition variable, but wait predicate does not need to
+// be protected by a mutex. Usage:
+// Waiting thread does:
+//
+// if (predicate)
+// return act();
+// Notifier::Waiter& w = waiters[my_index];
+// ec.prepare_wait(&w);
+// if (predicate) {
+// ec.cancel_wait(&w);
+// return act();
+// }
+// ec.commit_wait(&w);
+//
+// Notifying thread does:
+//
+// predicate = true;
+// ec.notify(true);
+//
+// notify is cheap if there are no waiting threads. prepare_wait/commit_wait are not
+// cheap, but they are executed only if the preceeding predicate check has
+// failed.
+//
+// Algorihtm outline:
+// There are two main variables: predicate (managed by user) and _state.
+// Operation closely resembles Dekker mutual algorithm:
+// https://en.wikipedia.org/wiki/Dekker%27s_algorithm
+// Waiting thread sets _state then checks predicate, Notifying thread sets
+// predicate then checks _state. Due to seq_cst fences in between these
+// operations it is guaranteed than either waiter will see predicate change
+// and won't block, or notifying thread will see _state change and will unblock
+// the waiter, or both. But it can't happen that both threads don't see each
+// other changes, which would lead to deadlock.
+class Notifier {
+
+ friend class Executor;
+
+ public:
+
+ struct Waiter {
+ std::atomic<Waiter*> next;
+ std::mutex mu;
+ std::condition_variable cv;
+ uint64_t epoch;
+ unsigned state;
+ enum {
+ kNotSignaled,
+ kWaiting,
+ kSignaled,
+ };
+ };
+
+ explicit Notifier(size_t N) : _waiters{N} {
+ assert(_waiters.size() < (1 << kWaiterBits) - 1);
+ // Initialize epoch to something close to overflow to test overflow.
+ _state = kStackMask | (kEpochMask - kEpochInc * _waiters.size() * 2);
+ }
+
+ ~Notifier() {
+ // Ensure there are no waiters.
+ assert((_state.load() & (kStackMask | kWaiterMask)) == kStackMask);
+ }
+
+ // prepare_wait prepares for waiting.
+ // After calling this function the thread must re-check the wait predicate
+ // and call either cancel_wait or commit_wait passing the same Waiter object.
+ void prepare_wait(Waiter* w) {
+ w->epoch = _state.fetch_add(kWaiterInc, std::memory_order_relaxed);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ }
+
+ // commit_wait commits waiting.
+ void commit_wait(Waiter* w) {
+ w->state = Waiter::kNotSignaled;
+ // Modification epoch of this waiter.
+ uint64_t epoch =
+ (w->epoch & kEpochMask) +
+ (((w->epoch & kWaiterMask) >> kWaiterShift) << kEpochShift);
+ uint64_t state = _state.load(std::memory_order_seq_cst);
+ for (;;) {
+ if (int64_t((state & kEpochMask) - epoch) < 0) {
+ // The preceeding waiter has not decided on its fate. Wait until it
+ // calls either cancel_wait or commit_wait, or is notified.
+ std::this_thread::yield();
+ state = _state.load(std::memory_order_seq_cst);
+ continue;
+ }
+ // We've already been notified.
+ if (int64_t((state & kEpochMask) - epoch) > 0) return;
+ // Remove this thread from prewait counter and add it to the waiter list.
+ assert((state & kWaiterMask) != 0);
+ uint64_t newstate = state - kWaiterInc + kEpochInc;
+ newstate = (newstate & ~kStackMask) | (w - &_waiters[0]);
+ if ((state & kStackMask) == kStackMask)
+ w->next.store(nullptr, std::memory_order_relaxed);
+ else
+ w->next.store(&_waiters[state & kStackMask], std::memory_order_relaxed);
+ if (_state.compare_exchange_weak(state, newstate,
+ std::memory_order_release))
+ break;
+ }
+ _park(w);
+ }
+
+ // cancel_wait cancels effects of the previous prepare_wait call.
+ void cancel_wait(Waiter* w) {
+ uint64_t epoch =
+ (w->epoch & kEpochMask) +
+ (((w->epoch & kWaiterMask) >> kWaiterShift) << kEpochShift);
+ uint64_t state = _state.load(std::memory_order_relaxed);
+ for (;;) {
+ if (int64_t((state & kEpochMask) - epoch) < 0) {
+ // The preceeding waiter has not decided on its fate. Wait until it
+ // calls either cancel_wait or commit_wait, or is notified.
+ std::this_thread::yield();
+ state = _state.load(std::memory_order_relaxed);
+ continue;
+ }
+ // We've already been notified.
+ if (int64_t((state & kEpochMask) - epoch) > 0) return;
+ // Remove this thread from prewait counter.
+ assert((state & kWaiterMask) != 0);
+ if (_state.compare_exchange_weak(state, state - kWaiterInc + kEpochInc,
+ std::memory_order_relaxed))
+ return;
+ }
+ }
+
+ // notify wakes one or all waiting threads.
+ // Must be called after changing the associated wait predicate.
+ void notify(bool all) {
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ uint64_t state = _state.load(std::memory_order_acquire);
+ for (;;) {
+ // Easy case: no waiters.
+ if ((state & kStackMask) == kStackMask && (state & kWaiterMask) == 0)
+ return;
+ uint64_t waiters = (state & kWaiterMask) >> kWaiterShift;
+ uint64_t newstate;
+ if (all) {
+ // Reset prewait counter and empty wait list.
+ newstate = (state & kEpochMask) + (kEpochInc * waiters) + kStackMask;
+ } else if (waiters) {
+ // There is a thread in pre-wait state, unblock it.
+ newstate = state + kEpochInc - kWaiterInc;
+ } else {
+ // Pop a waiter from list and unpark it.
+ Waiter* w = &_waiters[state & kStackMask];
+ Waiter* wnext = w->next.load(std::memory_order_relaxed);
+ uint64_t next = kStackMask;
+ if (wnext != nullptr) next = wnext - &_waiters[0];
+ // Note: we don't add kEpochInc here. ABA problem on the lock-free stack
+ // can't happen because a waiter is re-pushed onto the stack only after
+ // it was in the pre-wait state which inevitably leads to epoch
+ // increment.
+ newstate = (state & kEpochMask) + next;
+ }
+ if (_state.compare_exchange_weak(state, newstate,
+ std::memory_order_acquire)) {
+ if (!all && waiters) return; // unblocked pre-wait thread
+ if ((state & kStackMask) == kStackMask) return;
+ Waiter* w = &_waiters[state & kStackMask];
+ if (!all) w->next.store(nullptr, std::memory_order_relaxed);
+ _unpark(w);
+ return;
+ }
+ }
+ }
+
+ // notify n workers
+ void notify_n(size_t n) {
+ if(n >= _waiters.size()) {
+ notify(true);
+ }
+ else {
+ for(size_t k=0; k<n; ++k) {
+ notify(false);
+ }
+ }
+ }
+
+ size_t size() const {
+ return _waiters.size();
+ }
+
+ private:
+
+ // State_ layout:
+ // - low kStackBits is a stack of waiters committed wait.
+ // - next kWaiterBits is count of waiters in prewait state.
+ // - next kEpochBits is modification counter.
+ static const uint64_t kStackBits = 16;
+ static const uint64_t kStackMask = (1ull << kStackBits) - 1;
+ static const uint64_t kWaiterBits = 16;
+ static const uint64_t kWaiterShift = 16;
+ static const uint64_t kWaiterMask = ((1ull << kWaiterBits) - 1)
+ << kWaiterShift;
+ static const uint64_t kWaiterInc = 1ull << kWaiterBits;
+ static const uint64_t kEpochBits = 32;
+ static const uint64_t kEpochShift = 32;
+ static const uint64_t kEpochMask = ((1ull << kEpochBits) - 1) << kEpochShift;
+ static const uint64_t kEpochInc = 1ull << kEpochShift;
+ std::atomic<uint64_t> _state;
+ std::vector<Waiter> _waiters;
+
+ void _park(Waiter* w) {
+ std::unique_lock<std::mutex> lock(w->mu);
+ while (w->state != Waiter::kSignaled) {
+ w->state = Waiter::kWaiting;
+ w->cv.wait(lock);
+ }
+ }
+
+ void _unpark(Waiter* waiters) {
+ Waiter* next = nullptr;
+ for (Waiter* w = waiters; w; w = next) {
+ next = w->next.load(std::memory_order_relaxed);
+ unsigned state;
+ {
+ std::unique_lock<std::mutex> lock(w->mu);
+ state = w->state;
+ w->state = Waiter::kSignaled;
+ }
+ // Avoid notifying if it wasn't waiting.
+ if (state == Waiter::kWaiting) w->cv.notify_one();
+ }
+ }
+
+ Notifier(const Notifier&) = delete;
+ Notifier& operator=(const Notifier&) = delete;
+
+ Notifier(Notifier&& rhs) :
+ _state {rhs._state.load()},
+ _waiters {std::move(rhs._waiters)} {
+ }
+
+
+};
+
+
+
+} // namespace tf ------------------------------------------------------------
+
--- /dev/null
+// 2020/04/30 - midified by Tsung-Wei Huang
+// - adding TaskflowBoard support
+//
+// 2019/07/31 - modified by Tsung-Wei Huang
+// - fixed the missing comma in outputing JSON
+//
+// 2019/06/13 - modified by Tsung-Wei Huang
+// - added TaskView interface
+//
+// 2019/04/17 - created by Tsung-Wei Huang
+
+#pragma once
+
+#include "task.hpp"
+
+namespace tf {
+
+/**
+@class: ObserverInterface
+
+@brief The interface class for creating an executor observer.
+
+The tf::ExecutorObserver class let users define methods to monitor the behaviors
+of an executor.
+This is particularly useful when you want to inspect the performance of an executor.
+*/
+class ObserverInterface {
+
+ friend class Executor;
+
+ public:
+
+ /**
+ @brief virtual destructor
+ */
+ virtual ~ObserverInterface() = default;
+
+ /**
+ @brief constructor-like method to call when the executor observer is fully created
+ @param num_workers the number of the worker threads in the executor
+ */
+ virtual void set_up(size_t num_workers) = 0;
+
+ /**
+ @brief method to call before a worker thread executes a closure
+ @param worker_id the id of this worker thread
+ @param task_view a constant wrapper object to the task
+ */
+ virtual void on_entry(size_t worker_id, TaskView task_view) = 0;
+
+ /**
+ @brief method to call after a worker thread executed a closure
+ @param worker_id the id of this worker thread
+ @param task_view a constant wrapper object to the task
+ */
+ virtual void on_exit(size_t worker_id, TaskView task_view) = 0;
+};
+
+// ----------------------------------------------------------------------------
+// ChromeTracingObserver definition
+// ----------------------------------------------------------------------------
+
+/**
+@class: ChromeTracingObserver
+
+@brief observer designed based on chrome tracing format
+
+*/
+class ChromeTracingObserver : public ObserverInterface {
+
+ friend class Executor;
+
+ // data structure to record each task execution
+ struct Segment {
+
+ std::string name;
+
+ std::chrono::time_point<std::chrono::steady_clock> beg;
+ std::chrono::time_point<std::chrono::steady_clock> end;
+
+ Segment(
+ const std::string& n,
+ std::chrono::time_point<std::chrono::steady_clock> b
+ );
+
+ Segment(
+ const std::string& n,
+ std::chrono::time_point<std::chrono::steady_clock> b,
+ std::chrono::time_point<std::chrono::steady_clock> e
+ );
+ };
+
+ // data structure to store the entire execution timeline
+ struct Timeline {
+ std::chrono::time_point<std::chrono::steady_clock> origin;
+ std::vector<std::vector<Segment>> segments;
+ };
+
+ public:
+
+ /**
+ @brief dump the timelines in JSON format to an ostream
+ @param ostream the target std::ostream to dump
+ */
+ inline void dump(std::ostream& ostream) const;
+
+ /**
+ @brief dump the timelines in JSON to a std::string
+ @return a JSON string
+ */
+ inline std::string dump() const;
+
+ /**
+ @brief clear the timeline data
+ */
+ inline void clear();
+
+ /**
+ @brief get the number of total tasks in the observer
+ @return number of total tasks
+ */
+ inline size_t num_tasks() const;
+
+ private:
+
+ inline void set_up(size_t num_workers) override final;
+ inline void on_entry(size_t worker_id, TaskView task_view) override final;
+ inline void on_exit(size_t worker_id, TaskView task_view) override final;
+
+ Timeline _timeline;
+};
+
+// constructor
+inline ChromeTracingObserver::Segment::Segment(
+ const std::string& n,
+ std::chrono::time_point<std::chrono::steady_clock> b
+) :
+ name {n}, beg {b} {
+}
+
+// constructor
+inline ChromeTracingObserver::Segment::Segment(
+ const std::string& n,
+ std::chrono::time_point<std::chrono::steady_clock> b,
+ std::chrono::time_point<std::chrono::steady_clock> e
+) :
+ name {n}, beg {b}, end {e} {
+}
+
+// Procedure: set_up
+inline void ChromeTracingObserver::set_up(size_t num_workers) {
+
+ _timeline.segments.resize(num_workers);
+
+ for(size_t w=0; w<num_workers; ++w) {
+ _timeline.segments[w].reserve(32);
+ }
+
+ _timeline.origin = std::chrono::steady_clock::now();
+}
+
+// Procedure: on_entry
+inline void ChromeTracingObserver::on_entry(size_t w, TaskView tv) {
+ _timeline.segments[w].emplace_back(
+ tv.name(), std::chrono::steady_clock::now()
+ );
+}
+
+// Procedure: on_exit
+inline void ChromeTracingObserver::on_exit(size_t w, TaskView) {
+ assert(_timeline.segments[w].size() > 0);
+ _timeline.segments[w].back().end = std::chrono::steady_clock::now();
+}
+
+// Function: clear
+inline void ChromeTracingObserver::clear() {
+ for(size_t w=0; w<_timeline.segments.size(); ++w) {
+ _timeline.segments[w].clear();
+ }
+}
+
+// Procedure: dump
+inline void ChromeTracingObserver::dump(std::ostream& os) const {
+
+ size_t first;
+
+ for(first = 0; first<_timeline.segments.size(); ++first) {
+ if(_timeline.segments[first].size() > 0) {
+ break;
+ }
+ }
+
+ os << '[';
+
+ for(size_t w=first; w<_timeline.segments.size(); w++) {
+
+ if(w != first && _timeline.segments[w].size() > 0) {
+ os << ',';
+ }
+
+ for(size_t i=0; i<_timeline.segments[w].size(); i++) {
+
+ os << '{'
+ << "\"cat\":\"ChromeTracingObserver\",";
+
+ // name field
+ os << "\"name\":\"";
+ if(_timeline.segments[w][i].name.empty()) {
+ os << w << '_' << i;
+ }
+ else {
+ os << _timeline.segments[w][i].name;
+ }
+ os << "\",";
+
+ // segment field
+ os << "\"ph\":\"X\","
+ << "\"pid\":1,"
+ << "\"tid\":" << w << ','
+ << "\"ts\":" << std::chrono::duration_cast<std::chrono::microseconds>(
+ _timeline.segments[w][i].beg - _timeline.origin
+ ).count() << ','
+ << "\"dur\":" << std::chrono::duration_cast<std::chrono::microseconds>(
+ _timeline.segments[w][i].end - _timeline.segments[w][i].beg
+ ).count();
+
+ if(i != _timeline.segments[w].size() - 1) {
+ os << "},";
+ }
+ else {
+ os << '}';
+ }
+ }
+ }
+ os << "]\n";
+}
+
+// Function: dump
+inline std::string ChromeTracingObserver::dump() const {
+ std::ostringstream oss;
+ dump(oss);
+ return oss.str();
+}
+
+// Function: num_tasks
+inline size_t ChromeTracingObserver::num_tasks() const {
+ return std::accumulate(
+ _timeline.segments.begin(), _timeline.segments.end(), size_t{0},
+ [](size_t sum, const auto& exe){
+ return sum + exe.size();
+ }
+ );
+}
+
+// ----------------------------------------------------------------------------
+// TFProfObserver definition
+// ----------------------------------------------------------------------------
+
+/**
+@class: TFProfObserver
+
+@brief observer designed based on taskflow board format
+
+*/
+class TFProfObserver : public ObserverInterface {
+
+ friend class Executor;
+
+ // data structure to record each task execution
+ struct Segment {
+
+ std::string name;
+ TaskType type;
+
+ std::chrono::time_point<std::chrono::steady_clock> beg;
+ std::chrono::time_point<std::chrono::steady_clock> end;
+
+ Segment(
+ const std::string& n,
+ TaskType t,
+ std::chrono::time_point<std::chrono::steady_clock> b
+ );
+
+ Segment(
+ const std::string& n,
+ TaskType t,
+ std::chrono::time_point<std::chrono::steady_clock> b,
+ std::chrono::time_point<std::chrono::steady_clock> e
+ );
+ };
+
+ // data structure to store the entire execution timeline
+ struct Timeline {
+ std::chrono::time_point<std::chrono::steady_clock> origin;
+ std::vector<std::vector<Segment>> segments;
+ };
+
+ public:
+
+ /**
+ @brief dump the timelines in JSON format to an ostream
+ @param ostream the target std::ostream to dump
+ */
+ inline void dump(std::ostream& ostream) const;
+
+ /**
+ @brief dump the timelines in JSON to a std::string
+ @return a JSON string
+ */
+ inline std::string dump() const;
+
+ /**
+ @brief clear the timeline data
+ */
+ inline void clear();
+
+ /**
+ @brief get the number of total tasks in the observer
+ @return number of total tasks
+ */
+ inline size_t num_tasks() const;
+
+ private:
+
+ inline void set_up(size_t num_workers) override final;
+ inline void on_entry(size_t worker_id, TaskView task_view) override final;
+ inline void on_exit(size_t worker_id, TaskView task_view) override final;
+
+ Timeline _timeline;
+
+ UUID _uuid;
+};
+
+// constructor
+inline TFProfObserver::Segment::Segment(
+ const std::string& n,
+ TaskType t,
+ std::chrono::time_point<std::chrono::steady_clock> b
+) :
+ name {n}, type {t}, beg {b} {
+}
+
+// constructor
+inline TFProfObserver::Segment::Segment(
+ const std::string& n,
+ TaskType t,
+ std::chrono::time_point<std::chrono::steady_clock> b,
+ std::chrono::time_point<std::chrono::steady_clock> e
+) :
+ name {n}, type {t}, beg {b}, end {e} {
+}
+
+// Procedure: set_up
+inline void TFProfObserver::set_up(size_t num_workers) {
+
+ _timeline.segments.resize(num_workers);
+
+ for(size_t w=0; w<num_workers; ++w) {
+ _timeline.segments[w].reserve(32);
+ }
+
+ _timeline.origin = std::chrono::steady_clock::now();
+}
+
+// Procedure: on_entry
+inline void TFProfObserver::on_entry(size_t w, TaskView tv) {
+ _timeline.segments[w].emplace_back(
+ tv.name(), tv.type(), std::chrono::steady_clock::now()
+ );
+}
+
+// Procedure: on_exit
+inline void TFProfObserver::on_exit(size_t w, TaskView) {
+ assert(_timeline.segments[w].size() > 0);
+ _timeline.segments[w].back().end = std::chrono::steady_clock::now();
+}
+
+// Function: clear
+inline void TFProfObserver::clear() {
+ for(size_t w=0; w<_timeline.segments.size(); ++w) {
+ _timeline.segments[w].clear();
+ }
+}
+
+// Procedure: dump
+inline void TFProfObserver::dump(std::ostream& os) const {
+
+ size_t first;
+
+ for(first = 0; first<_timeline.segments.size(); ++first) {
+ if(_timeline.segments[first].size() > 0) {
+ break;
+ }
+ }
+
+ // not timeline data to dump
+ if(first == _timeline.segments.size()) {
+ os << "{}\n";
+ return;
+ }
+
+ os << "{\"executor\":\"" << _uuid << "\",\"data\":[";
+
+ for(size_t w=first; w<_timeline.segments.size(); w++) {
+
+ if(_timeline.segments[w].empty()) {
+ continue;
+ }
+
+ if(w != first) {
+ os << ',';
+ }
+
+ os << "{\"worker\":\"worker " << w << "\",\"data\":[";
+ for(size_t i=0; i<_timeline.segments[w].size(); ++i) {
+
+ const auto& s = _timeline.segments[w][i];
+
+ if(i) os << ',';
+
+ // span
+ os << "{\"span\":["
+ << std::chrono::duration_cast<std::chrono::microseconds>(
+ s.beg - _timeline.origin
+ ).count() << ","
+ << std::chrono::duration_cast<std::chrono::microseconds>(
+ s.end - _timeline.origin
+ ).count() << "],";
+
+ // name
+ os << "\"name\":\"";
+ if(s.name.empty()) {
+ os << w << '_' << i;
+ }
+ else {
+ os << s.name;
+ }
+ os << "\",";
+
+ // category "type": "Condition Task",
+ os << "\"type\":\"" << task_type_to_string(s.type) << "\"";
+
+ os << "}";
+ }
+ os << "]}";
+ }
+
+ os << "]}\n";
+}
+
+// Function: dump
+inline std::string TFProfObserver::dump() const {
+ std::ostringstream oss;
+ dump(oss);
+ return oss.str();
+}
+
+// Function: num_tasks
+inline size_t TFProfObserver::num_tasks() const {
+ return std::accumulate(
+ _timeline.segments.begin(), _timeline.segments.end(), size_t{0},
+ [](size_t sum, const auto& exe){
+ return sum + exe.size();
+ }
+ );
+}
+
+// ----------------------------------------------------------------------------
+// Identifier for Each Built-in Observer
+// ----------------------------------------------------------------------------
+
+/** @enum ObserverType
+
+built-in observer types
+
+*/
+enum ObserverType {
+ TFPROF = 1,
+ CHROME = 2
+};
+
+/**
+@brief convert an observer type to a human-readable string
+*/
+inline const char* observer_type_to_string(ObserverType type) {
+ const char* val;
+ switch(type) {
+ case TFPROF: val = "TFProf"; break;
+ case CHROME: val = "Chrome"; break;
+ default: val = "undefined"; break;
+ }
+ return val;
+}
+
+// ----------------------------------------------------------------------------
+// Legacy Alias
+// ----------------------------------------------------------------------------
+using ExecutorObserverInterface = ObserverInterface;
+using ExecutorObserver = ChromeTracingObserver;
+
+
+} // end of namespace tf -----------------------------------------------------
+
+
--- /dev/null
+#pragma once
+
+#include "graph.hpp"
+
+namespace tf {
+
+// ----------------------------------------------------------------------------
+// Task Types
+// ----------------------------------------------------------------------------
+
+/**
+@enum TaskType
+
+@brief enumeration of all task types
+*/
+enum TaskType {
+ PLACEHOLDER_TASK = Node::PLACEHOLDER_WORK,
+#ifdef TF_ENABLE_CUDA
+ CUDAFLOW_TASK = Node::CUDAFLOW_WORK,
+#endif
+ STATIC_TASK = Node::STATIC_WORK,
+ DYNAMIC_TASK = Node::DYNAMIC_WORK,
+ CONDITION_TASK = Node::CONDITION_WORK,
+ MODULE_TASK = Node::MODULE_WORK,
+ NUM_TASK_TYPES
+};
+
+/**
+@brief convert a task type to a human-readable string
+*/
+inline const char* task_type_to_string(TaskType type) {
+
+ const char* val;
+
+ switch(type) {
+ case PLACEHOLDER_TASK: val = "placeholder"; break;
+#ifdef TF_ENABLE_CUDA
+ case CUDAFLOW_TASK: val = "cudaflow"; break;
+#endif
+ case STATIC_TASK: val = "static"; break;
+ case DYNAMIC_TASK: val = "subflow"; break;
+ case CONDITION_TASK: val = "condition"; break;
+ case MODULE_TASK: val = "module"; break;
+ default: val = "undefined"; break;
+ }
+
+ return val;
+}
+
+// ----------------------------------------------------------------------------
+// Task Traits
+// ----------------------------------------------------------------------------
+
+/**
+@struct is_static_task
+
+@brief determines if a callable is a static task
+
+A static task is a callable object constructible from std::function<void()>.
+*/
+template <typename C>
+constexpr bool is_static_task_v = is_invocable_r_v<void, C> &&
+ !is_invocable_r_v<int, C>;
+
+/**
+@struct is_dynamic_task
+
+@brief determines if a callable is a dynamic task
+
+A dynamic task is a callable object constructible from std::function<void(Subflow&)>.
+*/
+template <typename C>
+constexpr bool is_dynamic_task_v = is_invocable_r_v<void, C, Subflow&>;
+
+/**
+@struct is_condition_task
+
+@brief determines if a callable is a condition task
+
+A condition task is a callable object constructible from std::function<int()>.
+*/
+template <typename C>
+constexpr bool is_condition_task_v = is_invocable_r_v<int, C>;
+
+#ifdef TF_ENABLE_CUDA
+/**
+@struct is_cudaflow_task
+
+@brief determines if a callable is a cudaflow task
+
+A cudaFlow task is a callable object constructible from std::function<void(cudaFlow&)>.
+*/
+template <typename C>
+constexpr bool is_cudaflow_task_v = is_invocable_r_v<void, C, cudaFlow&>;
+#endif
+
+
+
+// ----------------------------------------------------------------------------
+// Task
+// ----------------------------------------------------------------------------
+
+/**
+@class Task
+
+@brief handle to a node in a task dependency graph
+
+A Task is handle object of a node in a dependency graph.
+It provides a set of methods for users to access and modify the attributes of
+the associated graph node.
+
+*/
+class Task {
+
+ friend class FlowBuilder;
+ friend class Taskflow;
+ friend class TaskView;
+
+ public:
+
+ /**
+ @brief constructs an empty task
+ */
+ Task() = default;
+
+ /**
+ @brief constructs the task with the copy of the other task
+ */
+ Task(const Task& other);
+
+ /**
+ @brief replaces the contents with a copy of the other task
+ */
+ Task& operator = (const Task&);
+
+ /**
+ @brief replaces the contents with a null pointer
+ */
+ Task& operator = (std::nullptr_t);
+
+ /**
+ @brief compares if two tasks are associated with the same graph node
+ */
+ bool operator == (const Task& rhs) const;
+
+ /**
+ @brief compares if two tasks are not associated with the same graph node
+ */
+ bool operator != (const Task& rhs) const;
+
+ /**
+ @brief queries the name of the task
+ */
+ const std::string& name() const;
+
+ /**
+ @brief queries the number of successors of the task
+ */
+ size_t num_successors() const;
+
+ /**
+ @brief queries the number of predecessors of the task
+ */
+ size_t num_dependents() const;
+
+ /**
+ @brief queries the number of strong dependents of the task
+ */
+ size_t num_strong_dependents() const;
+
+ /**
+ @brief queries the number of weak dependents of the task
+ */
+ size_t num_weak_dependents() const;
+
+ /**
+ @brief assigns a name to the task
+
+ @param name a @std_string acceptable string
+
+ @return @c *this
+ */
+ Task& name(const std::string& name);
+
+ /**
+ @brief assigns a static task
+
+ @tparam C callable object type
+
+ @param callable a callable object constructible from std::function<void()>
+
+ @return @c *this
+ */
+ template <typename C>
+ std::enable_if_t<is_static_task_v<C>, Task>& work(C&& callable);
+
+ /**
+ @brief assigns a dynamic task
+
+ @tparam C callable object type
+
+ @param callable a callable object constructible from std::function<void(Subflow&)>
+
+ @return @c *this
+ */
+ template <typename C>
+ std::enable_if_t<is_dynamic_task_v<C>, Task>& work(C&& callable);
+
+ /**
+ @brief assigns a condition task
+
+ @tparam C callable object type
+
+ @param callable a callable object constructible from std::function<int()>
+
+ @return @c *this
+ */
+ template <typename C>
+ std::enable_if_t<is_condition_task_v<C>, Task>& work(C&& callable);
+
+#ifdef TF_ENABLE_CUDA
+ /**
+ @brief assigns a cudaFlow task
+
+ @tparam C callable object type
+
+ @param callable a callable object constructible from std::function<void(cudaFlow&)>
+
+ @return @c *this
+ */
+ template <typename C>
+ std::enable_if_t<is_cudaflow_task_v<C>, Task>& work(C&& callable);
+#endif
+
+ /**
+ @brief creates a module task from a taskflow
+
+ @param taskflow a taskflow object for the module
+
+ @return @c *this
+ */
+ Task& composed_of(Taskflow& taskflow);
+
+ /**
+ @brief adds precedence links from this to other tasks
+
+ @tparam Ts... parameter pack
+
+ @param tasks one or multiple tasks
+
+ @return @c *this
+ */
+ template <typename... Ts>
+ Task& precede(Ts&&... tasks);
+
+ /**
+ @brief adds precedence links from other tasks to this
+
+ @tparam Ts parameter pack
+
+ @param tasks one or multiple tasks
+
+ @return @c *this
+ */
+ template <typename... Ts>
+ Task& succeed(Ts&&... tasks);
+
+ /**
+ @brief resets the task handle to null
+ */
+ void reset();
+
+ /**
+ @brief resets the associated work to a placeholder
+ */
+ void reset_work();
+
+ /**
+ @brief queries if the task handle points to a task node
+ */
+ bool empty() const;
+
+ /**
+ @brief queries if the task has a work assigned
+ */
+ bool has_work() const;
+
+ /**
+ @brief applies an visitor callable to each successor of the task
+ */
+ template <typename V>
+ void for_each_successor(V&& visitor) const;
+
+ /**
+ @brief applies an visitor callable to each dependents of the task
+ */
+ template <typename V>
+ void for_each_dependent(V&& visitor) const;
+
+ /**
+ @brief obtains a hash value of the underlying node
+ */
+ size_t hash_value() const;
+
+ /**
+ @brief returns the task type
+ */
+ TaskType type() const;
+
+ private:
+
+ Task(Node*);
+
+ Node* _node {nullptr};
+
+ template <typename T>
+ void _precede(T&&);
+
+ template <typename T, typename... Rest>
+ void _precede(T&&, Rest&&...);
+
+ template <typename T>
+ void _succeed(T&&);
+
+ template <typename T, typename... Rest>
+ void _succeed(T&&, Rest&&...);
+};
+
+// Constructor
+inline Task::Task(Node* node) : _node {node} {
+}
+
+// Constructor
+inline Task::Task(const Task& rhs) : _node {rhs._node} {
+}
+
+// Function: precede
+template <typename... Ts>
+Task& Task::precede(Ts&&... tasks) {
+ //(_node->_precede(tgts._node), ...);
+ _precede(std::forward<Ts>(tasks)...);
+ return *this;
+}
+
+/// @private
+// Procedure: _precede
+template <typename T>
+void Task::_precede(T&& other) {
+ _node->_precede(other._node);
+}
+
+/// @private
+// Procedure: _precede
+template <typename T, typename... Ts>
+void Task::_precede(T&& task, Ts&&... others) {
+ _precede(std::forward<T>(task));
+ _precede(std::forward<Ts>(others)...);
+}
+
+// Function: succeed
+template <typename... Ts>
+Task& Task::succeed(Ts&&... tasks) {
+ //(tasks._node->_precede(_node), ...);
+ _succeed(std::forward<Ts>(tasks)...);
+ return *this;
+}
+
+/// @private
+// Procedure: succeed
+template <typename T>
+void Task::_succeed(T&& other) {
+ other._node->_precede(_node);
+}
+
+/// @private
+// Procedure: _succeed
+template <typename T, typename... Ts>
+void Task::_succeed(T&& task, Ts&&... others) {
+ _succeed(std::forward<T>(task));
+ _succeed(std::forward<Ts>(others)...);
+}
+
+// Function: composed_of
+inline Task& Task::composed_of(Taskflow& tf) {
+ _node->_handle.emplace<Node::ModuleWork>(&tf);
+ return *this;
+}
+
+// Operator =
+inline Task& Task::operator = (const Task& rhs) {
+ _node = rhs._node;
+ return *this;
+}
+
+// Operator =
+inline Task& Task::operator = (std::nullptr_t ptr) {
+ _node = ptr;
+ return *this;
+}
+
+// Operator ==
+inline bool Task::operator == (const Task& rhs) const {
+ return _node == rhs._node;
+}
+
+// Operator !=
+inline bool Task::operator != (const Task& rhs) const {
+ return _node != rhs._node;
+}
+
+// Function: name
+inline Task& Task::name(const std::string& name) {
+ _node->_name = name;
+ return *this;
+}
+
+// Procedure: reset
+inline void Task::reset() {
+ _node = nullptr;
+}
+
+// Procedure: reset_work
+inline void Task::reset_work() {
+ _node->_handle = nstd::monostate{};
+}
+
+// Function: name
+inline const std::string& Task::name() const {
+ return _node->_name;
+}
+
+// Function: num_dependents
+inline size_t Task::num_dependents() const {
+ return _node->num_dependents();
+}
+
+// Function: num_strong_dependents
+inline size_t Task::num_strong_dependents() const {
+ return _node->num_strong_dependents();
+}
+
+// Function: num_weak_dependents
+inline size_t Task::num_weak_dependents() const {
+ return _node->num_weak_dependents();
+}
+
+// Function: num_successors
+inline size_t Task::num_successors() const {
+ return _node->num_successors();
+}
+
+// Function: empty
+inline bool Task::empty() const {
+ return _node == nullptr;
+}
+
+// Function: has_work
+inline bool Task::has_work() const {
+ return _node ? _node->_handle.index() != 0 : false;
+}
+
+// Function: task_type
+inline TaskType Task::type() const {
+ return static_cast<TaskType>(_node->_handle.index());
+}
+
+// Function: for_each_successor
+template <typename V>
+void Task::for_each_successor(V&& visitor) const {
+ for(size_t i=0; i<_node->_successors.size(); ++i) {
+ visitor(Task(_node->_successors[i]));
+ }
+}
+
+// Function: for_each_dependent
+template <typename V>
+void Task::for_each_dependent(V&& visitor) const {
+ for(size_t i=0; i<_node->_dependents.size(); ++i) {
+ visitor(Task(_node->_dependents[i]));
+ }
+}
+
+// Function: hash_value
+inline size_t Task::hash_value() const {
+ return std::hash<Node*>{}(_node);
+}
+
+// Function: work
+// assign a static work
+template <typename C>
+std::enable_if_t<is_static_task_v<C>, Task>& Task::work(C&& c) {
+ _node->_handle.emplace<Node::StaticWork>(std::forward<C>(c));
+ return *this;
+}
+
+// Function: work
+// assigns a dynamic work
+template <typename C>
+std::enable_if_t<is_dynamic_task_v<C>, Task>& Task::work(C&& c) {
+ _node->_handle.emplace<Node::DynamicWork>(std::forward<C>(c));
+ return *this;
+}
+
+// Function: work
+// assigns a condition work
+template <typename C>
+std::enable_if_t<is_condition_task_v<C>, Task>& Task::work(C&& c) {
+ _node->_handle.emplace<Node::ConditionWork>(std::forward<C>(c));
+ return *this;
+}
+
+#ifdef TF_ENABLE_CUDA
+// Function: work
+// assigns a cudaFlow work
+template <typename C>
+std::enable_if_t<is_cudaflow_task_v<C>, Task>& Task::work(C&& c) {
+ _node->_handle.emplace<Node::cudaFlowWork>(std::forward<C>(c));
+ return *this;
+}
+#endif
+
+// ----------------------------------------------------------------------------
+
+/**
+@class TaskView
+
+@brief class to access task information from the observer interface
+*/
+class TaskView {
+
+ friend class Executor;
+
+ public:
+
+ /**
+ @brief constructs an empty task view
+ */
+ TaskView() = default;
+
+ /**
+ @brief constructs a task view from a task
+ */
+ TaskView(const Task& task);
+
+ /**
+ @brief constructs the task with the copy of the other task
+ */
+ TaskView(const TaskView& other);
+
+ /**
+ @brief replaces the contents with a copy of the other task
+ */
+ TaskView& operator = (const TaskView& other);
+
+ /**
+ @brief replaces the contents with another task
+ */
+ TaskView& operator = (const Task& other);
+
+ /**
+ @brief replaces the contents with a null pointer
+ */
+ TaskView& operator = (std::nullptr_t);
+
+ /**
+ @brief compares if two taskviews are associated with the same task
+ */
+ bool operator == (const TaskView&) const;
+
+ /**
+ @brief compares if two taskviews are associated with different tasks
+ */
+ bool operator != (const TaskView&) const;
+
+ /**
+ @brief queries the name of the task
+ */
+ const std::string& name() const;
+
+ /**
+ @brief queries the number of successors of the task
+ */
+ size_t num_successors() const;
+
+ /**
+ @brief queries the number of predecessors of the task
+ */
+ size_t num_dependents() const;
+
+ /**
+ @brief queries the number of strong dependents of the task
+ */
+ size_t num_strong_dependents() const;
+
+ /**
+ @brief queries the number of weak dependents of the task
+ */
+ size_t num_weak_dependents() const;
+
+ /**
+ @brief resets to an empty view
+ */
+ void reset();
+
+ /**
+ @brief queries if the task view is empty
+ */
+ bool empty() const;
+
+ /**
+ @brief applies an visitor callable to each successor of the task
+ */
+ template <typename V>
+ void for_each_successor(V&& visitor) const;
+
+ /**
+ @brief applies an visitor callable to each dependents of the task
+ */
+ template <typename V>
+ void for_each_dependent(V&& visitor) const;
+
+ /**
+ @brief queries the task type
+ */
+ TaskType type() const;
+
+ private:
+
+ TaskView(Node*);
+
+ Node* _node {nullptr};
+};
+
+// Constructor
+inline TaskView::TaskView(Node* node) : _node {node} {
+}
+
+// Constructor
+inline TaskView::TaskView(const TaskView& rhs) : _node {rhs._node} {
+}
+
+// Constructor
+inline TaskView::TaskView(const Task& task) : _node {task._node} {
+}
+
+// Operator =
+inline TaskView& TaskView::operator = (const TaskView& rhs) {
+ _node = rhs._node;
+ return *this;
+}
+
+// Operator =
+inline TaskView& TaskView::operator = (const Task& rhs) {
+ _node = rhs._node;
+ return *this;
+}
+
+// Operator =
+inline TaskView& TaskView::operator = (std::nullptr_t ptr) {
+ _node = ptr;
+ return *this;
+}
+
+// Function: name
+inline const std::string& TaskView::name() const {
+ return _node->_name;
+}
+
+// Function: num_dependents
+inline size_t TaskView::num_dependents() const {
+ return _node->num_dependents();
+}
+
+// Function: num_strong_dependents
+inline size_t TaskView::num_strong_dependents() const {
+ return _node->num_strong_dependents();
+}
+
+// Function: num_weak_dependents
+inline size_t TaskView::num_weak_dependents() const {
+ return _node->num_weak_dependents();
+}
+
+// Function: num_successors
+inline size_t TaskView::num_successors() const {
+ return _node->num_successors();
+}
+
+// Function: reset
+inline void TaskView::reset() {
+ _node = nullptr;
+}
+
+// Function: empty
+inline bool TaskView::empty() const {
+ return _node == nullptr;
+}
+
+// Function: type
+inline TaskType TaskView::type() const {
+ return static_cast<TaskType>(_node->_handle.index());
+}
+
+// Operator ==
+inline bool TaskView::operator == (const TaskView& rhs) const {
+ return _node == rhs._node;
+}
+
+// Operator !=
+inline bool TaskView::operator != (const TaskView& rhs) const {
+ return _node != rhs._node;
+}
+
+// Function: for_each_successor
+template <typename V>
+void TaskView::for_each_successor(V&& visitor) const {
+ for(size_t i=0; i<_node->_successors.size(); ++i) {
+ visitor(TaskView(_node->_successors[i]));
+ }
+}
+
+// Function: for_each_dependent
+template <typename V>
+void TaskView::for_each_dependent(V&& visitor) const {
+ for(size_t i=0; i<_node->_dependents.size(); ++i) {
+ visitor(TaskView(_node->_dependents[i]));
+ }
+}
+
+} // end of namespace tf. ---------------------------------------------------
+
+namespace std {
+
+/**
+@class hash<tf::Task>
+
+@brief hash specialization for std::hash<tf::Task>
+
+*/
+template <>
+struct hash<tf::Task> {
+ auto operator() (const tf::Task& task) const noexcept {
+ return task.hash_value();
+ }
+};
+
+} // end of namespace std ----------------------------------------------------
+
+
+
--- /dev/null
+#pragma once
+
+#include <stack>
+
+#include "flow_builder.hpp"
+#include "topology.hpp"
+
+namespace tf {
+
+// ----------------------------------------------------------------------------
+
+/**
+@class Taskflow
+
+@brief main entry to create a task dependency graph
+
+*/
+class Taskflow : public FlowBuilder {
+
+ friend class Topology;
+ friend class Executor;
+ friend class FlowBuilder;
+
+ struct Dumper {
+ std::stack<const Taskflow*> stack;
+ std::unordered_set<const Taskflow*> visited;
+ };
+
+ public:
+
+ /**
+ @brief constructs a taskflow with a given name
+ */
+ Taskflow(const std::string& name);
+
+ /**
+ @brief constructs a taskflow
+ */
+ Taskflow();
+
+ /**
+ @brief destroy the taskflow (virtual call)
+ */
+ virtual ~Taskflow();
+
+ /**
+ @brief dumps the taskflow to a std::ostream in DOT format
+
+ @param ostream a std::ostream target
+ */
+ void dump(std::ostream& ostream) const;
+
+ /**
+ @brief dumps the taskflow in DOT format to a std::string
+ */
+ std::string dump() const;
+
+ /**
+ @brief queries the number of tasks in the taskflow
+ */
+ size_t num_tasks() const;
+
+ /**
+ @brief queries the emptiness of the taskflow
+ */
+ bool empty() const;
+
+ /**
+ @brief sets the name of the taskflow
+
+ @return @c *this
+ */
+ void name(const std::string&);
+
+ /**
+ @brief queries the name of the taskflow
+ */
+ const std::string& name() const ;
+
+ /**
+ @brief clears the associated task dependency graph
+ */
+ void clear();
+
+ /**
+ @brief applies an visitor callable to each task in the taskflow
+ */
+ template <typename V>
+ void for_each_task(V&& visitor) const;
+
+ private:
+
+ std::string _name;
+
+ Graph _graph;
+
+ std::mutex _mtx;
+
+ std::list<Topology> _topologies;
+
+ void _dump(std::ostream&, const Taskflow*) const;
+ void _dump(std::ostream&, const Node*, Dumper&) const;
+ void _dump(std::ostream&, const Graph&, Dumper&) const;
+};
+
+// Constructor
+inline Taskflow::Taskflow(const std::string& name) :
+ FlowBuilder {_graph},
+ _name {name} {
+}
+
+// Constructor
+inline Taskflow::Taskflow() : FlowBuilder{_graph} {
+}
+
+// Destructor
+inline Taskflow::~Taskflow() {
+ assert(_topologies.empty());
+}
+
+// Procedure:
+inline void Taskflow::clear() {
+ _graph.clear();
+}
+
+// Function: num_noces
+inline size_t Taskflow::num_tasks() const {
+ return _graph.size();
+}
+
+// Function: empty
+inline bool Taskflow::empty() const {
+ return _graph.empty();
+}
+
+// Function: name
+inline void Taskflow::name(const std::string &name) {
+ _name = name;
+}
+
+// Function: name
+inline const std::string& Taskflow::name() const {
+ return _name;
+}
+
+// Function: for_each_task
+template <typename V>
+void Taskflow::for_each_task(V&& visitor) const {
+ for(size_t i=0; i<_graph._nodes.size(); ++i) {
+ visitor(Task(_graph._nodes[i]));
+ }
+}
+
+// Procedure: dump
+inline std::string Taskflow::dump() const {
+ std::ostringstream oss;
+ dump(oss);
+ return oss.str();
+}
+
+// Function: dump
+inline void Taskflow::dump(std::ostream& os) const {
+ os << "digraph Taskflow {\n";
+ _dump(os, this);
+ os << "}\n";
+}
+
+// Procedure: _dump
+inline void Taskflow::_dump(std::ostream& os, const Taskflow* top) const {
+
+ Dumper dumper;
+
+ dumper.stack.push(top);
+ dumper.visited.insert(top);
+
+ while(!dumper.stack.empty()) {
+
+ auto f = dumper.stack.top();
+ dumper.stack.pop();
+
+ os << "subgraph cluster_p" << f << " {\nlabel=\"Taskflow: ";
+ if(f->_name.empty()) os << 'p' << f;
+ else os << f->_name;
+ os << "\";\n";
+ _dump(os, f->_graph, dumper);
+ os << "}\n";
+ }
+}
+
+// Procedure: _dump
+inline void Taskflow::_dump(
+ std::ostream& os, const Node* node, Dumper& dumper
+) const {
+
+ os << 'p' << node << "[label=\"";
+ if(node->_name.empty()) os << 'p' << node;
+ else os << node->_name;
+ os << "\" ";
+
+ // shape for node
+ switch(node->_handle.index()) {
+
+ case Node::CONDITION_WORK:
+ os << "shape=diamond color=black fillcolor=aquamarine style=filled";
+ break;
+
+#ifdef TF_ENABLE_CUDA
+ case Node::CUDAFLOW_WORK:
+ os << "shape=folder fillcolor=cyan style=filled";
+ break;
+#endif
+
+ default:
+ break;
+ }
+
+ os << "];\n";
+
+ for(size_t s=0; s<node->_successors.size(); ++s) {
+ if(node->_handle.index() == Node::CONDITION_WORK) {
+ // case edge is dashed
+ os << 'p' << node << " -> p" << node->_successors[s]
+ << " [style=dashed label=\"" << s << "\"];\n";
+ }
+ else {
+ os << 'p' << node << " -> p" << node->_successors[s] << ";\n";
+ }
+ }
+
+ // subflow join node
+ if(node->_parent && node->_successors.size() == 0) {
+ os << 'p' << node << " -> p" << node->_parent << ";\n";
+ }
+
+ switch(node->_handle.index()) {
+
+ case Node::DYNAMIC_WORK: {
+ auto& sbg = nstd::get<Node::DynamicWork>(node->_handle).subgraph;
+ if(!sbg.empty()) {
+ os << "subgraph cluster_p" << node << " {\nlabel=\"Subflow: ";
+ if(node->_name.empty()) os << 'p' << node;
+ else os << node->_name;
+
+ os << "\";\n" << "color=blue\n";
+ _dump(os, sbg, dumper);
+ os << "}\n";
+ }
+ }
+ break;
+
+#ifdef TF_ENABLE_CUDA
+ case Node::CUDAFLOW_WORK: {
+ auto& cfg = nstd::get<Node::cudaFlowWork>(node->_handle).graph;
+ if(!cfg.empty()) {
+ os << "subgraph cluster_p" << node << " {\nlabel=\"cudaFlow: ";
+ if(node->_name.empty()) os << 'p' << node;
+ else os << node->_name;
+
+ os << "\";\n" << "color=\"purple\"\n";
+
+ for(const auto& v : cfg._nodes) {
+
+ os << 'p' << v.get() << "[label=\"";
+ if(v->_name.empty()) {
+ os << 'p' << v.get() << "\"";
+ }
+ else {
+ os << v->_name << "\"";
+ }
+
+ switch(v->_handle.index()) {
+ case cudaNode::NOOP:
+ break;
+
+ case cudaNode::COPY:
+ //os << " shape=\"cds\"";
+ break;
+
+ case cudaNode::KERNEL:
+ os << " style=\"filled\""
+ << " color=\"white\" fillcolor=\"black\""
+ << " fontcolor=\"white\""
+ << " shape=\"box3d\"";
+ break;
+
+ default:
+ break;
+ }
+
+ os << "];\n";
+ for(const auto s : v->_successors) {
+ os << 'p' << v.get() << " -> " << 'p' << s << ";\n";
+ }
+
+ if(v->_successors.size() == 0) {
+ os << 'p' << v.get() << " -> p" << node << ";\n";
+ }
+
+ }
+ os << "}\n";
+ }
+ }
+ break;
+#endif
+
+ default:
+ break;
+ }
+}
+
+// Procedure: _dump
+inline void Taskflow::_dump(
+ std::ostream& os, const Graph& graph, Dumper& dumper
+) const {
+
+ for(const auto& n : graph._nodes) {
+
+ // regular task
+ if(n->_handle.index() != Node::MODULE_WORK) {
+ _dump(os, n, dumper);
+ }
+ // module task
+ else {
+
+ auto module = nstd::get<Node::ModuleWork>(n->_handle).module;
+
+ os << 'p' << n << "[shape=box3d, color=blue, label=\"";
+ if(n->_name.empty()) os << n;
+ else os << n->_name;
+ os << " [Taskflow: ";
+ if(module->_name.empty()) os << 'p' << module;
+ else os << module->_name;
+ os << "]\"];\n";
+
+ if(dumper.visited.find(module) == dumper.visited.end()) {
+ dumper.visited.insert(module);
+ dumper.stack.push(module);
+ }
+
+ for(const auto s : n->_successors) {
+ os << 'p' << n << "->" << 'p' << s << ";\n";
+ }
+ }
+ }
+}
+
+// ----------------------------------------------------------------------------
+// Backward compatibility
+// ----------------------------------------------------------------------------
+using Framework = Taskflow;
+
+} // end of namespace tf. ---------------------------------------------------
+
--- /dev/null
+#pragma once
+
+//#include "taskflow.hpp"
+
+namespace tf {
+
+// ----------------------------------------------------------------------------
+
+// class: Topology
+class Topology {
+
+ friend class Taskflow;
+ friend class Executor;
+
+ public:
+
+ template <typename P, typename C>
+ Topology(Taskflow&, P&&, C&&);
+
+ private:
+
+ Taskflow& _taskflow;
+
+ std::promise<void> _promise;
+
+ PassiveVector<Node*> _sources;
+
+ std::function<bool()> _pred;
+ std::function<void()> _call;
+
+ std::atomic<size_t> _join_counter {0};
+};
+
+// Constructor
+template <typename P, typename C>
+inline Topology::Topology(Taskflow& tf, P&& p, C&& c):
+ _taskflow(tf),
+ _pred {std::forward<P>(p)},
+ _call {std::forward<C>(c)} {
+}
+
+// Procedure: _bind
+// Re-builds the source links and the sink number for this topology.
+//inline void Topology::_bind(Graph& g) {
+//
+// _sources.clear();
+//
+// //PassiveVector<Node*> condition_nodes;
+//
+// // scan each node in the graph and build up the links
+// for(auto& node : g.nodes()) {
+//
+// node->_topology = this;
+// node->_clear_state();
+// node->_set_up_join_counter();
+//
+// if(node->num_dependents() == 0) {
+// _sources.push_back(node.get());
+// }
+//
+// //int join_counter = 0;
+// //for(auto p : node->_dependents) {
+// // if(p->_work.index() == Node::CONDITION_WORK) {
+// // node->_set_state(Node::BRANCH);
+// // }
+// // else {
+// // join_counter++;
+// // }
+// //}
+//
+// //node->_join_counter.store(join_counter, std::memory_order_relaxed);
+//
+// //// TODO: Merge with the loop below?
+// //if(node->_work.index() == Node::CONDITION_WORK) {
+// // condition_nodes.push_back(node.get());
+// //}
+//
+// //// Reset each node's num_dependents
+// //node->_join_counter.store(node->_dependents.size(), std::memory_order_relaxed);
+// }
+//
+// // We need to deduct the condition predecessors in impure case nodes
+// //for(auto& n: condition_nodes) {
+// // for(auto& s: n->_successors) {
+// // s->_join_counter.fetch_sub(1, std::memory_order_relaxed);
+// // s->set_branch();
+// // }
+// //}
+//}
+
+
+
+} // end of namespace tf. ----------------------------------------------------
--- /dev/null
+// 2020/02/24 - created by twhuang
+// - specialized work stealing queue for pointer
+
+#pragma once
+
+#include <atomic>
+#include <vector>
+#include <cassert>
+#include <cstdint>
+#include <cstddef>
+#include <cstdlib>
+
+namespace tf {
+
+/**
+@class: TaskQueue
+
+@tparam T data type (must be a pointer)
+
+@brief Lock-free unbounded single-producer multiple-consumer queue.
+
+This class implements the work stealing queue described in the paper,
+"Correct and Efficient Work-Stealing for Weak Memory Models,"
+available at https://www.di.ens.fr/~zappa/readings/ppopp13.pdf.
+
+Only the queue owner can perform pop and push operations,
+while others can steal data from the queue.
+*/
+template <typename T>
+class TaskQueue {
+
+ static_assert(std::is_pointer<T>::value, "T must be a pointer type");
+
+ struct Array {
+
+ int64_t C;
+ int64_t M;
+ std::atomic<T>* S;
+
+ explicit Array(int64_t c) :
+ C {c},
+ M {c-1},
+ S {new std::atomic<T>[static_cast<size_t>(C)]} {
+ }
+
+ ~Array() {
+ delete [] S;
+ }
+
+ int64_t capacity() const noexcept {
+ return C;
+ }
+
+ template <typename O>
+ void push(int64_t i, O&& o) noexcept {
+ S[i & M].store(std::forward<O>(o), std::memory_order_relaxed);
+ }
+
+ T pop(int64_t i) noexcept {
+ return S[i & M].load(std::memory_order_relaxed);
+ }
+
+ Array* resize(int64_t b, int64_t t) {
+ Array* ptr = new Array {2*C};
+ for(int64_t i=t; i!=b; ++i) {
+ ptr->push(i, pop(i));
+ }
+ return ptr;
+ }
+
+ };
+
+ std::atomic<int64_t> _top;
+ std::atomic<int64_t> _bottom;
+ std::atomic<Array*> _array;
+ std::vector<Array*> _garbage;
+
+ public:
+
+ /**
+ @brief constructs the queue with a given capacity
+
+ @param capacity the capacity of the queue (must be power of 2)
+ */
+ explicit TaskQueue(int64_t capacity = 1024);
+
+ /**
+ @brief destructs the queue
+ */
+ ~TaskQueue();
+
+ /**
+ @brief queries if the queue is empty at the time of this call
+ */
+ bool empty() const noexcept;
+
+ /**
+ @brief queries the number of items at the time of this call
+ */
+ size_t size() const noexcept;
+
+ /**
+ @brief queries the capacity of the queue
+ */
+ int64_t capacity() const noexcept;
+
+ /**
+ @brief inserts an item to the queue
+
+ Only the owner thread can insert an item to the queue.
+ The operation can trigger the queue to resize its capacity
+ if more space is required.
+
+ @tparam O data type
+
+ @param item the item to perfect-forward to the queue
+ */
+ void push(T item);
+
+ /**
+ @brief pops out an item from the queue
+
+ Only the owner thread can pop out an item from the queue.
+ The return can be a nullptr if this operation failed (empty queue).
+ */
+ T pop();
+
+ /**
+ @brief steals an item from the queue
+
+ Any threads can try to steal an item from the queue.
+ The return can be a nullptr if this operation failed (not necessary empty).
+ */
+ T steal();
+};
+
+// Constructor
+template <typename T>
+TaskQueue<T>::TaskQueue(int64_t c) {
+ assert(c && (!(c & (c-1))));
+ _top.store(0, std::memory_order_relaxed);
+ _bottom.store(0, std::memory_order_relaxed);
+ _array.store(new Array{c}, std::memory_order_relaxed);
+ _garbage.reserve(32);
+}
+
+// Destructor
+template <typename T>
+TaskQueue<T>::~TaskQueue() {
+ for(auto a : _garbage) {
+ delete a;
+ }
+ delete _array.load();
+}
+
+// Function: empty
+template <typename T>
+bool TaskQueue<T>::empty() const noexcept {
+ int64_t b = _bottom.load(std::memory_order_relaxed);
+ int64_t t = _top.load(std::memory_order_relaxed);
+ return b <= t;
+}
+
+// Function: size
+template <typename T>
+size_t TaskQueue<T>::size() const noexcept {
+ int64_t b = _bottom.load(std::memory_order_relaxed);
+ int64_t t = _top.load(std::memory_order_relaxed);
+ return static_cast<size_t>(b >= t ? b - t : 0);
+}
+
+// Function: push
+template <typename T>
+void TaskQueue<T>::push(T o) {
+ int64_t b = _bottom.load(std::memory_order_relaxed);
+ int64_t t = _top.load(std::memory_order_acquire);
+ Array* a = _array.load(std::memory_order_relaxed);
+
+ // queue is full
+ if(a->capacity() - 1 < (b - t)) {
+ Array* tmp = a->resize(b, t);
+ _garbage.push_back(a);
+ std::swap(a, tmp);
+ _array.store(a, std::memory_order_relaxed);
+ }
+
+ a->push(b, o);
+ std::atomic_thread_fence(std::memory_order_release);
+ _bottom.store(b + 1, std::memory_order_relaxed);
+}
+
+// Function: pop
+template <typename T>
+T TaskQueue<T>::pop() {
+ int64_t b = _bottom.load(std::memory_order_relaxed) - 1;
+ Array* a = _array.load(std::memory_order_relaxed);
+ _bottom.store(b, std::memory_order_relaxed);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ int64_t t = _top.load(std::memory_order_relaxed);
+
+ T item {nullptr};
+
+ if(t <= b) {
+ item = a->pop(b);
+ if(t == b) {
+ // the last item just got stolen
+ if(!_top.compare_exchange_strong(t, t+1,
+ std::memory_order_seq_cst,
+ std::memory_order_relaxed)) {
+ item = nullptr;
+ }
+ _bottom.store(b + 1, std::memory_order_relaxed);
+ }
+ }
+ else {
+ _bottom.store(b + 1, std::memory_order_relaxed);
+ }
+
+ return item;
+}
+
+// Function: steal
+template <typename T>
+T TaskQueue<T>::steal() {
+ int64_t t = _top.load(std::memory_order_acquire);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ int64_t b = _bottom.load(std::memory_order_acquire);
+
+ T item {nullptr};
+
+ if(t < b) {
+ Array* a = _array.load(std::memory_order_consume);
+ item = a->pop(t);
+ if(!_top.compare_exchange_strong(t, t+1,
+ std::memory_order_seq_cst,
+ std::memory_order_relaxed)) {
+ return nullptr;
+ }
+ }
+
+ return item;
+}
+
+// Function: capacity
+template <typename T>
+int64_t TaskQueue<T>::capacity() const noexcept {
+ return _array.load(std::memory_order_relaxed)->capacity();
+}
+
+} // end of namespace tf -----------------------------------------------------
--- /dev/null
+#pragma once
+
+#include "cuda_error.hpp"
+
+namespace tf {
+
+/**
+@brief queries the number of available devices
+*/
+inline size_t cuda_num_devices() {
+ int N = 0;
+ TF_CHECK_CUDA(cudaGetDeviceCount(&N), "failed to get device count");
+ return static_cast<size_t>(N);
+}
+
+/**
+@brief gets the current device associated with the caller thread
+*/
+inline int cuda_get_device() {
+ int id;
+ TF_CHECK_CUDA(cudaGetDevice(&id), "failed to get current device id");
+ return id;
+}
+
+/**
+@brief switches to a given device context
+*/
+inline void cuda_set_device(int id) {
+ TF_CHECK_CUDA(cudaSetDevice(id), "failed to switch to device ", id);
+}
+
+/**
+@brief obtains the device property
+*/
+inline void cuda_get_device_property(int i, cudaDeviceProp& p) {
+ TF_CHECK_CUDA(
+ cudaGetDeviceProperties(&p, i), "failed to get property of device ", i
+ );
+}
+
+/**
+@brief obtains the device property
+*/
+inline cudaDeviceProp cuda_get_device_property(int i) {
+ cudaDeviceProp p;
+ TF_CHECK_CUDA(
+ cudaGetDeviceProperties(&p, i), "failed to get property of device ", i
+ );
+ return p;
+}
+
+/**
+@brief cuda_dump_device_property
+*/
+inline void cuda_dump_device_property(std::ostream& os, const cudaDeviceProp& p) {
+
+ os << "Major revision number: " << p.major << '\n'
+ << "Minor revision number: " << p.minor << '\n'
+ << "Name: " << p.name << '\n'
+ << "Total global memory: " << p.totalGlobalMem << '\n'
+ << "Total shared memory per block: " << p.sharedMemPerBlock << '\n'
+ << "Total registers per block: " << p.regsPerBlock << '\n'
+ << "Warp size: " << p.warpSize << '\n'
+ << "Maximum memory pitch: " << p.memPitch << '\n'
+ << "Maximum threads per block: " << p.maxThreadsPerBlock << '\n';
+
+ os << "Maximum dimension of block: ";
+ for (int i = 0; i < 3; ++i) {
+ if(i) os << 'x';
+ os << p.maxThreadsDim[i];
+ }
+ os << '\n';
+
+ os << "Maximum dimenstion of grid: ";
+ for (int i = 0; i < 3; ++i) {
+ if(i) os << 'x';
+ os << p.maxGridSize[i];;
+ }
+ os << '\n';
+
+ os << "Clock rate: " << p.clockRate << '\n'
+ << "Total constant memory: " << p.totalConstMem << '\n'
+ << "Texture alignment: " << p.textureAlignment << '\n'
+ << "Concurrent copy and execution: " << p.deviceOverlap << '\n'
+ << "Number of multiprocessors: " << p.multiProcessorCount << '\n'
+ << "Kernel execution timeout: " << p.kernelExecTimeoutEnabled << '\n'
+ << "GPU sharing Host Memory: " << p.integrated << '\n'
+ << "Host page-locked mem mapping: " << p.canMapHostMemory << '\n'
+ << "Alignment for Surfaces: " << p.surfaceAlignment << '\n'
+ << "Device has ECC support: " << p.ECCEnabled << '\n'
+ << "Unified Addressing (UVA): " << p.unifiedAddressing << '\n';
+}
+
+// ----------------------------------------------------------------------------
+// Class definitions
+// ----------------------------------------------------------------------------
+
+/** @class cudaScopedDevice
+
+@brief RAII-style device context switch
+
+*/
+class cudaScopedDevice {
+
+ public:
+
+ cudaScopedDevice(int);
+ ~cudaScopedDevice();
+
+ private:
+
+ int _p;
+};
+
+// Constructor
+inline cudaScopedDevice::cudaScopedDevice(int dev) {
+ TF_CHECK_CUDA(cudaGetDevice(&_p), "failed to get current device scope");
+ if(_p == dev) {
+ _p = -1;
+ }
+ else {
+ TF_CHECK_CUDA(cudaSetDevice(dev), "failed to scope on device ", dev);
+ }
+}
+
+// Destructor
+inline cudaScopedDevice::~cudaScopedDevice() {
+ if(_p != -1) {
+ cudaSetDevice(_p);
+ //TF_CHECK_CUDA(cudaSetDevice(_p), "failed to scope back to device ", _p);
+ }
+}
+
+} // end of namespace cuda ---------------------------------------------------
+
+
--- /dev/null
+#pragma once
+
+#include <cuda.h>
+#include <iostream>
+#include <sstream>
+#include <exception>
+
+#include "../utility/stringify.hpp"
+
+#define TF_CUDA_REMOVE_FIRST_HELPER(N, ...) __VA_ARGS__
+#define TF_CUDA_REMOVE_FIRST(...) TF_CUDA_REMOVE_FIRST_HELPER(__VA_ARGS__)
+#define TF_CUDA_GET_FIRST_HELPER(N, ...) N
+#define TF_CUDA_GET_FIRST(...) TF_CUDA_GET_FIRST_HELPER(__VA_ARGS__)
+
+#define TF_CHECK_CUDA(...) \
+if(TF_CUDA_GET_FIRST(__VA_ARGS__) != cudaSuccess) { \
+ std::ostringstream oss; \
+ auto ev = TF_CUDA_GET_FIRST(__VA_ARGS__); \
+ auto unknown_str = "unknown error"; \
+ auto unknown_name = "cudaErrorUnknown"; \
+ auto error_str = ::cudaGetErrorString(ev); \
+ auto error_name = ::cudaGetErrorName(ev); \
+ oss << "[" << __FILE__ << ":" << __LINE__ << "] " \
+ << (error_str ? error_str : unknown_str) \
+ << " (" \
+ << (error_name ? error_name : unknown_name) \
+ << ") - "; \
+ tf::ostreamize(oss, TF_CUDA_REMOVE_FIRST(__VA_ARGS__)); \
+ throw std::runtime_error(oss.str()); \
+}
+
--- /dev/null
+#pragma once
+
+#include "cuda_task.hpp"
+
+namespace tf {
+
+/**
+@class cudaFlow
+
+@brief methods for building a CUDA task dependency graph.
+
+A cudaFlow is a high-level interface to manipulate GPU tasks using
+the task dependency graph model.
+The class provides a set of methods for creating and launch different tasks
+on one or multiple CUDA devices,
+for instance, kernel tasks, data transfer tasks, and memory operation tasks.
+*/
+class cudaFlow {
+
+ friend class Executor;
+
+ public:
+
+ /**
+ @brief constructs a cudaFlow builder object
+
+ @tparam P predicate type
+
+ @param graph a cudaGraph to manipulate
+ @param p predicate which return @c true if the launching should be contined
+ */
+ template <typename P>
+ cudaFlow(cudaGraph& graph, P&& p);
+
+ /**
+ @brief queries the emptiness of the graph
+ */
+ bool empty() const;
+
+ /**
+ @brief creates a no-operation task
+
+ An empty node performs no operation during execution,
+ but can be used for transitive ordering.
+ For example, a phased execution graph with 2 groups of n nodes
+ with a barrier between them can be represented using an empty node
+ and 2*n dependency edges,
+ rather than no empty node and n^2 dependency edges.
+ */
+ cudaTask noop();
+
+ // CUDA seems pretty restrictive about calling host in a cudaGraph.
+ // We disable this function and wait for future stability.
+ //
+ //@brief creates a host execution task
+ //
+ //@tparam C callable type
+ //
+ //@param c a callable object constructible from std::function<void()>.
+
+ //A host can only execute CPU-specific functions and cannot do any CUDA calls
+ //(e.g., cudaMalloc).
+ //
+ //template <typename C>
+ //cudaTask host(C&& c);
+
+ /**
+ @brief creates a kernel task
+
+ @tparam F kernel function type
+ @tparam ArgsT kernel function parameters type
+
+ @param g configured grid
+ @param b configured block
+ @param s configured shared memory
+ @param f kernel function
+ @param args arguments to forward to the kernel function by copy
+
+ @return cudaTask handle
+ */
+ template <typename F, typename... ArgsT>
+ cudaTask kernel(dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args);
+
+ /**
+ @brief creates a kernel task on a device
+
+ @tparam F kernel function type
+ @tparam ArgsT kernel function parameters type
+
+ @param d device identifier to luanch the kernel
+ @param g configured grid
+ @param b configured block
+ @param s configured shared memory
+ @param f kernel function
+ @param args arguments to forward to the kernel function by copy
+
+ @return cudaTask handle
+ */
+ template <typename F, typename... ArgsT>
+ cudaTask kernel_on(int d, dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args);
+
+ /**
+ @brief creates a memset task
+
+ @param dst pointer to the destination device memory area
+ @param v value to set for each byte of specified memory
+ @param count size in bytes to set
+
+ A memset task fills the first @c count bytes of device memory area
+ pointed by @c dst with the byte value @c v.
+ */
+ cudaTask memset(void* dst, int v, size_t count);
+
+ /**
+ @brief creates a memcpy task
+
+ @param tgt pointer to the target memory block
+ @param src pointer to the source memory block
+ @param bytes bytes to copy
+
+ @return cudaTask handle
+
+ A memcpy task transfers @c bytes of data from a course location
+ to a target location. Direction can be arbitrary among CPUs and GPUs.
+ */
+ cudaTask memcpy(void* tgt, const void* src, size_t bytes);
+
+ /**
+ @brief creates a zero task that zeroes a typed memory block
+
+ @tparam T element type (size of @c T must be either 1, 2, or 4)
+ @param dst pointer to the destination device memory area
+ @param count number of elements
+
+ A zero task zeroes the first @c count elements of type @c T
+ in a device memory area pointed by @c dst.
+ */
+ template <typename T>
+ std::enable_if_t<
+ is_pod_v<T> && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4),
+ cudaTask
+ >
+ zero(T* dst, size_t count);
+
+ /**
+ @brief creates a fill task that fills a typed memory block with a value
+
+ @tparam T element type (size of @c T must be either 1, 2, or 4)
+ @param dst pointer to the destination device memory area
+ @param value value to fill for each element of type @c T
+ @param count number of elements
+
+ A fill task fills the first @c count elements of type @c T with @c value
+ in a device memory area pointed by @c dst.
+ The value to fill is interpreted in type @c T rather than byte.
+ */
+ template <typename T>
+ std::enable_if_t<
+ is_pod_v<T> && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4),
+ cudaTask
+ >
+ fill(T* dst, T value, size_t count);
+
+ /**
+ @brief creates a copy task
+
+ @tparam T element type (non-void)
+
+ @param tgt pointer to the target memory block
+ @param src pointer to the source memory block
+ @param num number of elements to copy
+
+ @return cudaTask handle
+
+ A copy task transfers <tt>num*sizeof(T)</tt> bytes of data from a source location
+ to a target location. Direction can be arbitrary among CPUs and GPUs.
+ */
+ template <
+ typename T,
+ std::enable_if_t<!std::is_same<T, void>::value, void>* = nullptr
+ >
+ cudaTask copy(T* tgt, const T* src, size_t num);
+
+ /**
+ @brief assigns a device to launch the cudaFlow
+
+ @param device target device identifier
+ */
+ void device(int device);
+
+ /**
+ @brief queries the device associated with the cudaFlow
+ */
+ int device() const;
+
+ /**
+ @brief assigns a stream to launch the cudaFlow
+
+ @param stream target stream identifier
+ */
+ void stream(cudaStream_t stream);
+
+ /**
+ @brief assigns a predicate to loop the cudaFlow until the predicate is satisfied
+
+ @tparam P predicate type
+ @param p predicate which return @c true if the launching should be contined
+
+ The execution of cudaFlow is equivalent to: <tt>while(!predicate()) { run cudaflow; }</tt>
+ */
+ template <typename P>
+ void predicate(P&& p);
+
+ /**
+ @brief repeats the execution of the cudaFlow by @c n times
+ */
+ void repeat(size_t n);
+
+ private:
+
+ cudaGraph& _graph;
+
+ int _device {0};
+
+ nstd::optional<cudaStream_t> _stream;
+
+ std::function<bool()> _predicate;
+};
+
+// Constructor
+template <typename P>
+cudaFlow::cudaFlow(cudaGraph& g, P&& p) :
+ _graph {g},
+ _predicate {std::forward<P>(p)} {
+}
+
+// Procedure: predicate
+template <typename P>
+void cudaFlow::predicate(P&& pred) {
+ _predicate = std::forward<P>(pred);
+}
+
+// Procedure: repeat
+inline void cudaFlow::repeat(size_t n) {
+ _predicate = [n] () mutable { return n-- == 0; };
+}
+
+// Function: empty
+inline bool cudaFlow::empty() const {
+ return _graph._nodes.empty();
+}
+
+// Procedure: device
+inline void cudaFlow::device(int d) {
+ _device = d;
+}
+
+// Function: device
+inline int cudaFlow::device() const {
+ return _device;
+}
+
+// Procedure: stream
+inline void cudaFlow::stream(cudaStream_t s) {
+ _stream = s;
+}
+
+// Function: noop
+inline cudaTask cudaFlow::noop() {
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Noop>{},
+ [](cudaGraph_t& graph, cudaGraphNode_t& node){
+ TF_CHECK_CUDA(
+ ::cudaGraphAddEmptyNode(&node, graph, nullptr, 0),
+ "failed to create a no-operation (empty) node"
+ );
+ }
+ );
+ return cudaTask(node);
+}
+
+//// Function: host
+//template <typename C>
+//cudaTask cudaFlow::host(C&& c) {
+// auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Host>{},
+// [c=std::forward<C>(c)](cudaGraph_t& graph, cudaGraphNode_t& node) mutable {
+// cudaHostNodeParams p;
+// p.fn = [] (void* data) { (*static_cast<C*>(data))(); };
+// p.userData = &c;
+// TF_CHECK_CUDA(
+// ::cudaGraphAddHostNode(&node, graph, nullptr, 0, &p),
+// "failed to create a host node"
+// );
+// }
+// );
+// return cudaTask(node);
+//}
+
+// Function: kernel
+template <typename F, typename... ArgsT>
+cudaTask cudaFlow::kernel(
+ dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args
+) {
+
+ using traits = function_traits<F>;
+
+ static_assert(traits::arity == sizeof...(ArgsT), "arity mismatches");
+
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Kernel>{},
+ [g, b, s, f=(void*)f, args...] (cudaGraph_t& graph, cudaGraphNode_t& node) {
+
+ cudaKernelNodeParams p;
+ void* arguments[sizeof...(ArgsT)] = { (void*)(&args)... };
+ p.func = f;
+ p.gridDim = g;
+ p.blockDim = b;
+ p.sharedMemBytes = s;
+ p.kernelParams = arguments;
+ p.extra = nullptr;
+
+ TF_CHECK_CUDA(
+ ::cudaGraphAddKernelNode(&node, graph, nullptr, 0, &p),
+ "failed to create a cudaGraph node in kernel task"
+ );
+ }
+ );
+
+ return cudaTask(node);
+}
+
+// Function: kernel
+template <typename F, typename... ArgsT>
+cudaTask cudaFlow::kernel_on(
+ int d, dim3 g, dim3 b, size_t s, F&& f, ArgsT&&... args
+) {
+
+ using traits = function_traits<F>;
+
+ static_assert(traits::arity == sizeof...(ArgsT), "arity mismatches");
+
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Kernel>{},
+ [d, g, b, s, f=(void*)f, args...] (cudaGraph_t& graph, cudaGraphNode_t& node) {
+
+ cudaKernelNodeParams p;
+ void* arguments[sizeof...(ArgsT)] = { (void*)(&args)... };
+ p.func = f;
+ p.gridDim = g;
+ p.blockDim = b;
+ p.sharedMemBytes = s;
+ p.kernelParams = arguments;
+ p.extra = nullptr;
+
+ cudaScopedDevice ctx(d);
+ TF_CHECK_CUDA(
+ ::cudaGraphAddKernelNode(&node, graph, nullptr, 0, &p),
+ "failed to create a cudaGraph node in kernel_on task"
+ );
+ }
+ );
+
+ return cudaTask(node);
+}
+
+// Function: zero
+template <typename T>
+std::enable_if_t<
+ is_pod_v<T> && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4),
+ cudaTask
+>
+cudaFlow::zero(T* dst, size_t count) {
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Memset>{},
+ [dst, count] (cudaGraph_t& graph, cudaGraphNode_t& node) {
+ cudaMemsetParams p;
+ p.dst = dst;
+ p.value = 0;
+ p.pitch = 0;
+ p.elementSize = sizeof(T); // either 1, 2, or 4
+ p.width = count;
+ p.height = 1;
+ TF_CHECK_CUDA(
+ cudaGraphAddMemsetNode(&node, graph, nullptr, 0, &p),
+ "failed to create a cudaGraph node in zero task"
+ );
+ }
+ );
+ return cudaTask(node);
+}
+
+// Function: fill
+template <typename T>
+std::enable_if_t<
+ is_pod_v<T> && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4),
+ cudaTask
+>
+cudaFlow::fill(T* dst, T value, size_t count) {
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Memset>{},
+ [dst, value, count] (cudaGraph_t& graph, cudaGraphNode_t& node) {
+ cudaMemsetParams p;
+ p.dst = dst;
+
+ // perform bit-wise copy
+ p.value = 0; // crucial
+ static_assert(sizeof(T) <= sizeof(p.value), "internal error");
+ std::memcpy(&p.value, &value, sizeof(T));
+
+ p.pitch = 0;
+ p.elementSize = sizeof(T); // either 1, 2, or 4
+ p.width = count;
+ p.height = 1;
+ TF_CHECK_CUDA(
+ cudaGraphAddMemsetNode(&node, graph, nullptr, 0, &p),
+ "failed to create a cudaGraph node in fill task"
+ );
+ }
+ );
+ return cudaTask(node);
+}
+
+// Function: copy
+template <
+ typename T,
+ std::enable_if_t<!std::is_same<T, void>::value, void>*
+>
+cudaTask cudaFlow::copy(T* tgt, const T* src, size_t num) {
+
+ using U = std::decay_t<T>;
+
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Copy>{},
+ [tgt, src, num] (cudaGraph_t& graph, cudaGraphNode_t& node) {
+
+ cudaMemcpy3DParms p;
+ p.srcArray = nullptr;
+ p.srcPos = ::make_cudaPos(0, 0, 0);
+ p.srcPtr = ::make_cudaPitchedPtr(const_cast<T*>(src), num*sizeof(U), num, 1);
+ p.dstArray = nullptr;
+ p.dstPos = ::make_cudaPos(0, 0, 0);
+ p.dstPtr = ::make_cudaPitchedPtr(tgt, num*sizeof(U), num, 1);
+ p.extent = ::make_cudaExtent(num*sizeof(U), 1, 1);
+ p.kind = cudaMemcpyDefault;
+
+ TF_CHECK_CUDA(
+ cudaGraphAddMemcpyNode(&node, graph, nullptr, 0, &p),
+ "failed to create a cudaGraph node in copy task"
+ );
+ }
+ );
+
+ return cudaTask(node);
+}
+
+// Function: memset
+inline cudaTask cudaFlow::memset(void* dst, int ch, size_t count) {
+
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Memset>{},
+ [dst, ch, count] (cudaGraph_t& graph, cudaGraphNode_t& node) {
+ cudaMemsetParams p;
+ p.dst = dst;
+ p.value = ch;
+ p.pitch = 0;
+ //p.elementSize = (count & 1) == 0 ? ((count & 3) == 0 ? 4 : 2) : 1;
+ //p.width = (count & 1) == 0 ? ((count & 3) == 0 ? count >> 2 : count >> 1) : count;
+ p.elementSize = 1; // either 1, 2, or 4
+ p.width = count;
+ p.height = 1;
+ TF_CHECK_CUDA(
+ cudaGraphAddMemsetNode(&node, graph, nullptr, 0, &p),
+ "failed to create a cudaGraph node in memset task"
+ );
+ }
+ );
+
+ return cudaTask(node);
+}
+
+// Function: memcpy
+inline cudaTask cudaFlow::memcpy(void* tgt, const void* src, size_t bytes) {
+ auto node = _graph.emplace_back(nstd::in_place_type_t<cudaNode::Copy>{},
+ [tgt, src, bytes] (cudaGraph_t& graph, cudaGraphNode_t& node) {
+ // Parameters in cudaPitchedPtr
+ // d - Pointer to allocated memory
+ // p - Pitch of allocated memory in bytes
+ // xsz - Logical width of allocation in elements
+ // ysz - Logical height of allocation in elements
+ cudaMemcpy3DParms p;
+ p.srcArray = nullptr;
+ p.srcPos = ::make_cudaPos(0, 0, 0);
+ p.srcPtr = ::make_cudaPitchedPtr(const_cast<void*>(src), bytes, bytes, 1);
+ p.dstArray = nullptr;
+ p.dstPos = ::make_cudaPos(0, 0, 0);
+ p.dstPtr = ::make_cudaPitchedPtr(tgt, bytes, bytes, 1);
+ p.extent = ::make_cudaExtent(bytes, 1, 1);
+ p.kind = cudaMemcpyDefault;
+ TF_CHECK_CUDA(
+ cudaGraphAddMemcpyNode(&node, graph, nullptr, 0, &p),
+ "failed to create a cudaGraph node in memcpy task"
+ );
+ }
+ );
+ return cudaTask(node);
+}
+
+} // end of namespace tf -----------------------------------------------------
+
+
--- /dev/null
+#pragma once
+
+#include "cuda_device.hpp"
+
+#include "../utility/object_pool.hpp"
+#include "../utility/traits.hpp"
+#include "../utility/passive_vector.hpp"
+#include "../nstd/variant.hpp"
+#include "../nstd/optional.hpp"
+
+namespace tf {
+
+// ----------------------------------------------------------------------------
+// cudaNode class
+// ----------------------------------------------------------------------------
+
+// class: cudaNode
+class cudaNode {
+
+ friend class cudaFlow;
+ friend class cudaGraph;
+ friend class cudaTask;
+
+ friend class Taskflow;
+ friend class Executor;
+
+
+ // Noop handle
+ struct Noop {
+
+ template <typename C>
+ Noop(C&&);
+
+ std::function<void(cudaGraph_t&, cudaGraphNode_t&)> work;
+ };
+
+ //// Host handle
+ //struct Host {
+
+ // template <typename C>
+ // Host(C&&);
+ //
+ // std::function<void(cudaGraph_t&, cudaGraphNode_t&)> work;
+ //};
+
+ // Memset handle
+ struct Memset {
+
+ template <typename C>
+ Memset(C&&);
+
+ std::function<void(cudaGraph_t&, cudaGraphNode_t&)> work;
+ };
+
+ // Copy handle
+ struct Copy {
+
+ template <typename C>
+ Copy(C&&);
+
+ std::function<void(cudaGraph_t&, cudaGraphNode_t&)> work;
+ };
+
+ // Kernel handle
+ struct Kernel {
+
+ template <typename C>
+ Kernel(C&&);
+
+ std::function<void(cudaGraph_t&, cudaGraphNode_t&)> work;
+ };
+
+ using handle_t = nstd::variant<
+ nstd::monostate,
+ Noop,
+ //Host,
+ Memset,
+ Copy,
+ Kernel
+ >;
+
+ // variant index
+ constexpr static auto NOOP = get_index_v<Noop, handle_t>;
+ //constexpr static auto HOST = get_index_v<Host, handle_t>;
+ constexpr static auto MEMSET = get_index_v<Memset, handle_t>;
+ constexpr static auto COPY = get_index_v<Copy, handle_t>;
+ constexpr static auto KERNEL = get_index_v<Kernel, handle_t>;
+
+ public:
+
+ template <typename... ArgsT>
+ cudaNode(ArgsT&&...);
+
+ private:
+
+ std::string _name;
+
+ handle_t _handle;
+
+ cudaGraphNode_t _native_handle {nullptr};
+
+ PassiveVector<cudaNode*> _successors;
+
+ void _precede(cudaNode*);
+};
+
+// ----------------------------------------------------------------------------
+// cudaGraph class
+// ----------------------------------------------------------------------------
+
+// class: cudaGraph
+class cudaGraph {
+
+ friend class cudaFlow;
+ friend class cudaNode;
+ friend class cudaTask;
+
+ friend class Taskflow;
+ friend class Executor;
+
+ public:
+
+ ~cudaGraph();
+
+ template <typename... ArgsT>
+ cudaNode* emplace_back(ArgsT&&...);
+
+ cudaGraph_t native_handle();
+
+ void clear();
+
+ bool empty() const;
+
+ private:
+
+ cudaGraph_t _native_handle {nullptr};
+
+ std::vector<std::unique_ptr<cudaNode>> _nodes;
+
+ void _make_native_graph();
+};
+
+// ----------------------------------------------------------------------------
+// cudaNode definitions
+// ----------------------------------------------------------------------------
+
+//// Host handle constructor
+//template <typename C>
+//cudaNode::Host::Host(C&& c) : work {std::forward<C>(c)} {
+//}
+
+// Noop handle constructor
+template <typename C>
+cudaNode::Noop::Noop(C&& c) : work {std::forward<C>(c)} {
+}
+
+// Memset handle constructor
+template <typename C>
+cudaNode::Memset::Memset(C&& c) : work {std::forward<C>(c)} {
+}
+
+// Copy handle constructor
+template <typename C>
+cudaNode::Copy::Copy(C&& c) : work {std::forward<C>(c)} {
+}
+
+// Kernel handle constructor
+template <typename C>
+cudaNode::Kernel::Kernel(C&& c) : work {std::forward<C>(c)} {
+}
+
+// Constructor
+template <typename... ArgsT>
+cudaNode::cudaNode(ArgsT&&... args) : _handle {std::forward<ArgsT>(args)...} {
+}
+
+// Procedure: _precede
+inline void cudaNode::_precede(cudaNode* v) {
+ _successors.push_back(v);
+}
+
+// ----------------------------------------------------------------------------
+// cudaGraph definitions
+// ----------------------------------------------------------------------------
+
+// Destructor
+inline cudaGraph::~cudaGraph() {
+ if(_native_handle) {
+ cudaGraphDestroy(_native_handle);
+ }
+}
+
+// Function: empty
+inline bool cudaGraph::empty() const {
+ return _nodes.empty();
+}
+
+// Procedure: clear
+inline void cudaGraph::clear() {
+
+ _nodes.clear();
+
+ if(_native_handle) {
+ TF_CHECK_CUDA(
+ cudaGraphDestroy(_native_handle), "failed to destroy a cudaGraph on clear"
+ );
+ _native_handle = nullptr;
+ }
+}
+
+// Function: emplace_back
+template <typename... ArgsT>
+cudaNode* cudaGraph::emplace_back(ArgsT&&... args) {
+ auto node = std::make_unique<cudaNode>(std::forward<ArgsT>(args)...);
+ _nodes.emplace_back(std::move(node));
+ return _nodes.back().get();
+}
+
+// Function: native_handle
+inline cudaGraph_t cudaGraph::native_handle() {
+ return _native_handle;
+}
+
+// Procedure: _make_native_graph
+inline void cudaGraph::_make_native_graph() {
+
+ //// TODO: must be nullptr
+ //if(_native_handle) {
+ // TF_CHECK_CUDA(
+ // cudaGraphDestroy(_native_handle), "failed to destroy the previous cudaGraph"
+ // );
+ // _native_handle = nullptr;
+ //}
+ //
+ //cudaScopedDevice ctx {d};
+ assert(_native_handle == nullptr);
+
+ TF_CHECK_CUDA(
+ cudaGraphCreate(&_native_handle, 0), "failed to create a cudaGraph"
+ );
+
+ // create nodes
+ for(auto& node : _nodes) {
+ switch(node->_handle.index()) {
+ case cudaNode::NOOP:
+ nstd::get<cudaNode::Noop>(node->_handle).work(
+ _native_handle, node->_native_handle
+ );
+ break;
+
+ //case cudaNode::HOST:
+ // nstd::get<cudaNode::Host>(node->_handle).work(
+ // _native_handle, node->_native_handle
+ // );
+ //break;
+
+ case cudaNode::MEMSET:
+ nstd::get<cudaNode::Memset>(node->_handle).work(
+ _native_handle, node->_native_handle
+ );
+ break;
+
+ case cudaNode::COPY:
+ nstd::get<cudaNode::Copy>(node->_handle).work(
+ _native_handle, node->_native_handle
+ );
+ break;
+
+ case cudaNode::KERNEL:
+ nstd::get<cudaNode::Kernel>(node->_handle).work(
+ _native_handle, node->_native_handle
+ );
+ break;
+ }
+ }
+
+ // create edges
+ for(auto& node : _nodes) {
+ for(auto succ : node->_successors){
+ TF_CHECK_CUDA(
+ ::cudaGraphAddDependencies(
+ _native_handle, &(node->_native_handle), &(succ->_native_handle), 1
+ ),
+ "failed to add a preceding link"
+ );
+ }
+ }
+
+}
+
+
+//inline void cudaGraph::run() {
+// cudaGraphExec_t graphExec;
+// TF_CHECK_CUDA(
+// cudaGraphInstantiate(&graphExec, _handle, nullptr, nullptr, 0),
+// "failed to create an executable cudaGraph"
+// );
+// TF_CHECK_CUDA(cudaGraphLaunch(graphExec, 0), "failed to launch cudaGraph")
+// TF_CHECK_CUDA(cudaStreamSynchronize(0), "failed to sync cudaStream");
+// TF_CHECK_CUDA(
+// cudaGraphExecDestroy(graphExec), "failed to destroy an executable cudaGraph"
+// );
+//}
+
+
+
+
+
+} // end of namespace tf -----------------------------------------------------
+
--- /dev/null
+#pragma once
+
+#include "cuda_graph.hpp"
+
+namespace tf {
+
+/**
+@class cudaTask
+
+@brief handle to a node in a cudaGraph
+*/
+class cudaTask {
+
+ friend class cudaFlow;
+
+ public:
+
+ /**
+ @brief constructs an empty cudaTask
+ */
+ cudaTask() = default;
+
+ /**
+ @brief copy-constructs a cudaTask
+ */
+ cudaTask(const cudaTask&) = default;
+
+ /**
+ @brief copy-assigns a cudaTask
+ */
+ cudaTask& operator = (const cudaTask&) = default;
+
+ /**
+ @brief adds precedence links from this to other tasks
+
+ @tparam Ts... parameter pack
+
+ @param tasks one or multiple tasks
+
+ @return @c *this
+ */
+ template <typename... Ts>
+ cudaTask& precede(Ts&&... tasks);
+
+ /**
+ @brief adds precedence links from other tasks to this
+
+ @tparam Ts... parameter pack
+
+ @param tasks one or multiple tasks
+
+ @return @c *this
+ */
+ template <typename... Ts>
+ cudaTask& succeed(Ts&&... tasks);
+
+ /**
+ @brief assigns a name to the task
+
+ @param name a @std_string acceptable string
+
+ @return @c *this
+ */
+ cudaTask& name(const std::string& name);
+
+ /**
+ @brief queries the name of the task
+ */
+ const std::string& name() const;
+
+ /**
+ @brief queries the number of successors
+ */
+ size_t num_successors() const;
+
+ /**
+ @brief queries if the task is associated with a cudaNode
+ */
+ bool empty() const;
+
+ private:
+
+ cudaTask(cudaNode*);
+
+ cudaNode* _node {nullptr};
+
+ /// @private
+ template <typename T>
+ void _precede(T&&);
+
+ /// @private
+ template <typename T, typename... Ts>
+ void _precede(T&&, Ts&&...);
+
+ /// @private
+ template <typename T>
+ void _succeed(T&&);
+
+ // @private
+ template <typename T, typename... Ts>
+ void _succeed(T&&, Ts&&...);
+};
+
+// Constructor
+inline cudaTask::cudaTask(cudaNode* node) : _node {node} {
+}
+
+// Function: precede
+template <typename... Ts>
+cudaTask& cudaTask::precede(Ts&&... tasks) {
+ _precede(std::forward<Ts>(tasks)...);
+ return *this;
+}
+
+/// @private
+// Procedure: precede
+template <typename T>
+void cudaTask::_precede(T&& other) {
+ _node->_precede(other._node);
+}
+
+/// @private
+// Procedure: _precede
+template <typename T, typename... Ts>
+void cudaTask::_precede(T&& task, Ts&&... others) {
+ _precede(std::forward<T>(task));
+ _precede(std::forward<Ts>(others)...);
+}
+
+// Function: succeed
+template <typename... Ts>
+cudaTask& cudaTask::succeed(Ts&&... tasks) {
+ _succeed(std::forward<Ts>(tasks)...);
+ return *this;
+}
+
+/// @private
+// Procedure: _succeed
+template <typename T>
+void cudaTask::_succeed(T&& other) {
+ other._node->_precede(_node);
+}
+
+/// @private
+// Procedure: _succeed
+template <typename T, typename... Ts>
+void cudaTask::_succeed(T&& task, Ts&&... others) {
+ _succeed(std::forward<T>(task));
+ _succeed(std::forward<Ts>(others)...);
+}
+
+// Function: empty
+inline bool cudaTask::empty() const {
+ return _node == nullptr;
+}
+
+// Function: name
+inline cudaTask& cudaTask::name(const std::string& name) {
+ _node->_name = name;
+ return *this;
+}
+
+// Function: name
+inline const std::string& cudaTask::name() const {
+ return _node->_name;
+}
+
+// Function: num_successors
+inline size_t cudaTask::num_successors() const {
+ return _node->_successors.size();
+}
+
+} // end of namespace tf -----------------------------------------------------
--- /dev/null
+#pragma once
+
+namespace tf {
+
+// ----------------------------------------------------------------------------
+// forward declarations
+// ----------------------------------------------------------------------------
+
+// taskflow
+class Node;
+class Graph;
+class FlowBuilder;
+class Subflow;
+class Task;
+class TaskView;
+class Taskflow;
+class Topology;
+class Executor;
+class WorkerView;
+class ObserverInterface;
+class ChromeTracingObserver;
+class TFProfObserver;
+
+// cudaflow
+class cudaNode;
+class cudaGraph;
+class cudaTask;
+class cudaFlow;
+
+
+
+} // end of namespace tf -----------------------------------------------------
+
+
+
+
--- /dev/null
+//
+// Copyright (c) 2016-2018 Martin Moene
+//
+// https://github.com/martinmoene/any-lite
+//
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#ifndef NSTD_ANY_LITE_HPP
+#define NSTD_ANY_LITE_HPP
+
+#define any_lite_MAJOR 0
+#define any_lite_MINOR 2
+#define any_lite_PATCH 0
+
+#define any_lite_VERSION any_STRINGIFY(any_lite_MAJOR) "." any_STRINGIFY(any_lite_MINOR) "." any_STRINGIFY(any_lite_PATCH)
+
+#define any_STRINGIFY( x ) any_STRINGIFY_( x )
+#define any_STRINGIFY_( x ) #x
+
+// any-lite configuration:
+
+#define any_ANY_DEFAULT 0
+#define any_ANY_NSTD 1
+#define any_ANY_STD 2
+
+#if !defined( any_CONFIG_SELECT_ANY )
+# define any_CONFIG_SELECT_ANY ( any_HAVE_STD_ANY ? any_ANY_STD : any_ANY_NSTD )
+#endif
+
+// Control presence of exception handling (try and auto discover):
+
+#ifndef any_CONFIG_NO_EXCEPTIONS
+# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+# define any_CONFIG_NO_EXCEPTIONS 0
+# else
+# define any_CONFIG_NO_EXCEPTIONS 1
+# endif
+#endif
+
+// C++ language version detection (C++20 is speculative):
+// Note: VC14.0/1900 (VS2015) lacks too much from C++14.
+
+#ifndef any_CPLUSPLUS
+# if defined(_MSVC_LANG ) && !defined(__clang__)
+# define any_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG )
+# else
+# define any_CPLUSPLUS __cplusplus
+# endif
+#endif
+
+#define any_CPP98_OR_GREATER ( any_CPLUSPLUS >= 199711L )
+#define any_CPP11_OR_GREATER ( any_CPLUSPLUS >= 201103L )
+#define any_CPP14_OR_GREATER ( any_CPLUSPLUS >= 201402L )
+#define any_CPP17_OR_GREATER ( any_CPLUSPLUS >= 201703L )
+#define any_CPP20_OR_GREATER ( any_CPLUSPLUS >= 202000L )
+
+// Use C++17 std::any if available and requested:
+
+#if any_CPP17_OR_GREATER && defined(__has_include )
+# if __has_include( <any> )
+# define any_HAVE_STD_ANY 1
+# else
+# define any_HAVE_STD_ANY 0
+# endif
+#else
+# define any_HAVE_STD_ANY 0
+#endif
+
+#define any_USES_STD_ANY ( (any_CONFIG_SELECT_ANY == any_ANY_STD) || ((any_CONFIG_SELECT_ANY == any_ANY_DEFAULT) && any_HAVE_STD_ANY) )
+
+//
+// in_place: code duplicated in any-lite, expected-lite, optional-lite, value-ptr-lite, variant-lite:
+//
+
+#ifndef nstd_lite_HAVE_IN_PLACE_TYPES
+#define nstd_lite_HAVE_IN_PLACE_TYPES 1
+
+// C++17 std::in_place in <utility>:
+
+#if any_CPP17_OR_GREATER
+
+#include <utility>
+
+namespace nstd {
+
+using std::in_place;
+using std::in_place_type;
+using std::in_place_index;
+using std::in_place_t;
+using std::in_place_type_t;
+using std::in_place_index_t;
+
+#define nstd_lite_in_place_t( T) std::in_place_t
+#define nstd_lite_in_place_type_t( T) std::in_place_type_t<T>
+#define nstd_lite_in_place_index_t(K) std::in_place_index_t<K>
+
+#define nstd_lite_in_place( T) std::in_place_t{}
+#define nstd_lite_in_place_type( T) std::in_place_type_t<T>{}
+#define nstd_lite_in_place_index(K) std::in_place_index_t<K>{}
+
+} // namespace nstd
+
+#else // any_CPP17_OR_GREATER
+
+#include <cstddef>
+
+namespace nstd {
+namespace detail {
+
+template< class T >
+struct in_place_type_tag {};
+
+template< std::size_t K >
+struct in_place_index_tag {};
+
+} // namespace detail
+
+struct in_place_t {};
+
+template< class T >
+inline in_place_t in_place( detail::in_place_type_tag<T> = detail::in_place_type_tag<T>() )
+{
+ return in_place_t();
+}
+
+template< std::size_t K >
+inline in_place_t in_place( detail::in_place_index_tag<K> = detail::in_place_index_tag<K>() )
+{
+ return in_place_t();
+}
+
+template< class T >
+inline in_place_t in_place_type( detail::in_place_type_tag<T> = detail::in_place_type_tag<T>() )
+{
+ return in_place_t();
+}
+
+template< std::size_t K >
+inline in_place_t in_place_index( detail::in_place_index_tag<K> = detail::in_place_index_tag<K>() )
+{
+ return in_place_t();
+}
+
+// mimic templated typedef:
+
+#define nstd_lite_in_place_t( T) nstd::in_place_t(&)( nstd::detail::in_place_type_tag<T> )
+#define nstd_lite_in_place_type_t( T) nstd::in_place_t(&)( nstd::detail::in_place_type_tag<T> )
+#define nstd_lite_in_place_index_t(K) nstd::in_place_t(&)( nstd::detail::in_place_index_tag<K> )
+
+#define nstd_lite_in_place( T) nstd::in_place_type<T>
+#define nstd_lite_in_place_type( T) nstd::in_place_type<T>
+#define nstd_lite_in_place_index(K) nstd::in_place_index<K>
+
+} // namespace nstd
+
+#endif // any_CPP17_OR_GREATER
+#endif // nstd_lite_HAVE_IN_PLACE_TYPES
+
+//
+// Using std::any:
+//
+
+#if any_USES_STD_ANY
+
+#include <any>
+#include <utility>
+
+namespace nstd {
+
+ using std::any;
+ using std::any_cast;
+ using std::make_any;
+ using std::swap;
+ using std::bad_any_cast;
+}
+
+#else // any_USES_STD_ANY
+
+#include <utility>
+
+// Compiler versions:
+//
+// MSVC++ 6.0 _MSC_VER == 1200 any_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0)
+// MSVC++ 7.0 _MSC_VER == 1300 any_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002)
+// MSVC++ 7.1 _MSC_VER == 1310 any_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003)
+// MSVC++ 8.0 _MSC_VER == 1400 any_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005)
+// MSVC++ 9.0 _MSC_VER == 1500 any_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008)
+// MSVC++ 10.0 _MSC_VER == 1600 any_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010)
+// MSVC++ 11.0 _MSC_VER == 1700 any_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012)
+// MSVC++ 12.0 _MSC_VER == 1800 any_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013)
+// MSVC++ 14.0 _MSC_VER == 1900 any_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015)
+// MSVC++ 14.1 _MSC_VER >= 1910 any_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017)
+// MSVC++ 14.2 _MSC_VER >= 1920 any_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019)
+
+#if defined(_MSC_VER ) && !defined(__clang__)
+# define any_COMPILER_MSVC_VER (_MSC_VER )
+# define any_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * ( 5 + (_MSC_VER < 1900 ) ) )
+#else
+# define any_COMPILER_MSVC_VER 0
+# define any_COMPILER_MSVC_VERSION 0
+#endif
+
+#define any_COMPILER_VERSION( major, minor, patch ) ( 10 * ( 10 * (major) + (minor) ) + (patch) )
+
+#if defined(__clang__)
+# define any_COMPILER_CLANG_VERSION any_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__)
+#else
+# define any_COMPILER_CLANG_VERSION 0
+#endif
+
+#if defined(__GNUC__) && !defined(__clang__)
+# define any_COMPILER_GNUC_VERSION any_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
+#else
+# define any_COMPILER_GNUC_VERSION 0
+#endif
+
+// half-open range [lo..hi):
+//#define any_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) )
+
+// Presence of language and library features:
+
+#define any_HAVE( feature ) ( any_HAVE_##feature )
+
+#ifdef _HAS_CPP0X
+# define any_HAS_CPP0X _HAS_CPP0X
+#else
+# define any_HAS_CPP0X 0
+#endif
+
+#define any_CPP11_90 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1500)
+#define any_CPP11_100 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1600)
+#define any_CPP11_120 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1800)
+#define any_CPP11_140 (any_CPP11_OR_GREATER || any_COMPILER_MSVC_VER >= 1900)
+
+#define any_CPP14_000 (any_CPP14_OR_GREATER)
+#define any_CPP17_000 (any_CPP17_OR_GREATER)
+
+// Presence of C++11 language features:
+
+#define any_HAVE_CONSTEXPR_11 any_CPP11_140
+#define any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG \
+ any_CPP11_120
+#define any_HAVE_INITIALIZER_LIST any_CPP11_120
+#define any_HAVE_NOEXCEPT any_CPP11_140
+#define any_HAVE_NULLPTR any_CPP11_100
+#define any_HAVE_TYPE_TRAITS any_CPP11_90
+#define any_HAVE_STATIC_ASSERT any_CPP11_100
+#define any_HAVE_ADD_CONST any_CPP11_90
+#define any_HAVE_REMOVE_REFERENCE any_CPP11_90
+
+#define any_HAVE_TR1_ADD_CONST (!! any_COMPILER_GNUC_VERSION )
+#define any_HAVE_TR1_REMOVE_REFERENCE (!! any_COMPILER_GNUC_VERSION )
+#define any_HAVE_TR1_TYPE_TRAITS (!! any_COMPILER_GNUC_VERSION )
+
+// Presence of C++14 language features:
+
+#define any_HAVE_CONSTEXPR_14 any_CPP14_000
+
+// Presence of C++17 language features:
+
+#define any_HAVE_NODISCARD any_CPP17_000
+
+// Presence of C++ language features:
+
+#if any_HAVE_CONSTEXPR_11
+# define any_constexpr constexpr
+#else
+# define any_constexpr /*constexpr*/
+#endif
+
+#if any_HAVE_CONSTEXPR_14
+# define any_constexpr14 constexpr
+#else
+# define any_constexpr14 /*constexpr*/
+#endif
+
+#if any_HAVE_NOEXCEPT
+# define any_noexcept noexcept
+#else
+# define any_noexcept /*noexcept*/
+#endif
+
+#if any_HAVE_NULLPTR
+# define any_nullptr nullptr
+#else
+# define any_nullptr NULL
+#endif
+
+#if any_HAVE_NODISCARD
+# define any_nodiscard [[nodiscard]]
+#else
+# define any_nodiscard /*[[nodiscard]]*/
+#endif
+
+// additional includes:
+
+#if any_CONFIG_NO_EXCEPTIONS
+# include <cassert>
+#else
+# include <typeinfo>
+#endif
+
+#if ! any_HAVE_NULLPTR
+# include <cstddef>
+#endif
+
+#if any_HAVE_INITIALIZER_LIST
+# include <initializer_list>
+#endif
+
+#if any_HAVE_TYPE_TRAITS
+# include <type_traits>
+#elif any_HAVE_TR1_TYPE_TRAITS
+# include <tr1/type_traits>
+#endif
+
+// Method enabling
+
+#if any_CPP11_OR_GREATER
+
+#define any_REQUIRES_0(...) \
+ template< bool B = (__VA_ARGS__), typename std::enable_if<B, int>::type = 0 >
+
+#define any_REQUIRES_T(...) \
+ , typename = typename std::enable_if< (__VA_ARGS__), nstd::any_lite::detail::enabler >::type
+
+#define any_REQUIRES_R(R, ...) \
+ typename std::enable_if<__VA_ARGS__, R>::type
+
+#define any_REQUIRES_A(...) \
+ , typename std::enable_if<__VA_ARGS__, void*>::type = nullptr
+
+#endif
+
+//
+// any:
+//
+
+namespace nstd { namespace any_lite {
+
+// C++11 emulation:
+
+namespace std11 {
+
+#if any_HAVE_ADD_CONST
+
+using std::add_const;
+
+#elif any_HAVE_TR1_ADD_CONST
+
+using std::tr1::add_const;
+
+#else
+
+template< class T > struct add_const { typedef const T type; };
+
+#endif // any_HAVE_ADD_CONST
+
+#if any_HAVE_REMOVE_REFERENCE
+
+using std::remove_reference;
+
+#elif any_HAVE_TR1_REMOVE_REFERENCE
+
+using std::tr1::remove_reference;
+
+#else
+
+template< class T > struct remove_reference { typedef T type; };
+template< class T > struct remove_reference<T&> { typedef T type; };
+
+#endif // any_HAVE_REMOVE_REFERENCE
+
+} // namespace std11
+
+namespace detail {
+
+// for any_REQUIRES_T
+
+/*enum*/ class enabler{};
+
+} // namespace detail
+
+#if ! any_CONFIG_NO_EXCEPTIONS
+
+class bad_any_cast : public std::bad_cast
+{
+public:
+#if any_CPP11_OR_GREATER
+ virtual const char* what() const any_noexcept
+#else
+ virtual const char* what() const throw()
+#endif
+ {
+ return "any-lite: bad any_cast";
+ }
+};
+
+#endif // any_CONFIG_NO_EXCEPTIONS
+
+class any
+{
+public:
+ any_constexpr any() any_noexcept
+ : content( any_nullptr )
+ {}
+
+ any( any const & other )
+ : content( other.content ? other.content->clone() : any_nullptr )
+ {}
+
+#if any_CPP11_OR_GREATER
+
+ any( any && other ) any_noexcept
+ : content( std::move( other.content ) )
+ {
+ other.content = any_nullptr;
+ }
+
+ template<
+ class ValueType, class T = typename std::decay<ValueType>::type
+ any_REQUIRES_T( ! std::is_same<T, any>::value )
+ >
+ any( ValueType && value ) any_noexcept
+ : content( new holder<T>( std::forward<ValueType>( value ) ) )
+ {}
+
+ template<
+ class T, class... Args
+ any_REQUIRES_T( std::is_constructible<T, Args&&...>::value )
+ >
+ explicit any( nstd_lite_in_place_type_t(T), Args&&... args )
+ : content( new holder<T>( T( std::forward<Args>(args)... ) ) )
+ {}
+
+ template<
+ class T, class U, class... Args
+ any_REQUIRES_T( std::is_constructible<T, std::initializer_list<U>&, Args&&...>::value )
+ >
+ explicit any( nstd_lite_in_place_type_t(T), std::initializer_list<U> il, Args&&... args )
+ : content( new holder<T>( T( il, std::forward<Args>(args)... ) ) )
+ {}
+
+#else
+
+ template< class ValueType >
+ any( ValueType const & value )
+ : content( new holder<ValueType>( value ) )
+ {}
+
+#endif // any_CPP11_OR_GREATER
+
+ ~any()
+ {
+ reset();
+ }
+
+ any & operator=( any const & other )
+ {
+ any( other ).swap( *this );
+ return *this;
+ }
+
+#if any_CPP11_OR_GREATER
+
+ any & operator=( any && other ) any_noexcept
+ {
+ any( std::move( other ) ).swap( *this );
+ return *this;
+ }
+
+ template<
+ class ValueType, class T = typename std::decay<ValueType>::type
+ any_REQUIRES_T( ! std::is_same<T, any>::value )
+ >
+ any & operator=( ValueType && value )
+ {
+ any( std::move( value ) ).swap( *this );
+ return *this;
+ }
+
+ template< class T, class... Args >
+ void emplace( Args && ... args )
+ {
+ any( T( std::forward<Args>(args)... ) ).swap( *this );
+ }
+
+ template<
+ class T, class U, class... Args
+ any_REQUIRES_T( std::is_constructible<T, std::initializer_list<U>&, Args&&...>::value )
+ >
+ void emplace( std::initializer_list<U> il, Args&&... args )
+ {
+ any( T( il, std::forward<Args>(args)... ) ).swap( *this );
+ }
+
+#else
+
+ template< class ValueType >
+ any & operator=( ValueType const & value )
+ {
+ any( value ).swap( *this );
+ return *this;
+ }
+
+#endif // any_CPP11_OR_GREATER
+
+ void reset() any_noexcept
+ {
+ delete content; content = any_nullptr;
+ }
+
+ void swap( any & other ) any_noexcept
+ {
+ std::swap( content, other.content );
+ }
+
+ bool has_value() const any_noexcept
+ {
+ return content != any_nullptr;
+ }
+
+ const std::type_info & type() const any_noexcept
+ {
+ return has_value() ? content->type() : typeid( void );
+ }
+
+ //
+ // non-standard:
+ //
+
+ template< class ValueType >
+ const ValueType * to_ptr() const
+ {
+ return &( static_cast<holder<ValueType> *>( content )->held );
+ }
+
+ template< class ValueType >
+ ValueType * to_ptr()
+ {
+ return &( static_cast<holder<ValueType> *>( content )->held );
+ }
+
+private:
+ class placeholder
+ {
+ public:
+ virtual ~placeholder()
+ {
+ }
+
+ virtual std::type_info const & type() const = 0;
+
+ virtual placeholder * clone() const = 0;
+ };
+
+ template< typename ValueType >
+ class holder : public placeholder
+ {
+ public:
+ holder( ValueType const & value )
+ : held( value )
+ {}
+
+#if any_CPP11_OR_GREATER
+ holder( ValueType && value )
+ : held( std::move( value ) )
+ {}
+#endif
+
+ virtual std::type_info const & type() const
+ {
+ return typeid( ValueType );
+ }
+
+ virtual placeholder * clone() const
+ {
+ return new holder( held );
+ }
+
+ ValueType held;
+ };
+
+ placeholder * content;
+};
+
+inline void swap( any & x, any & y ) any_noexcept
+{
+ x.swap( y );
+}
+
+#if any_CPP11_OR_GREATER
+
+template< class T, class ...Args >
+inline any make_any( Args&& ...args )
+{
+ return any( nstd_lite_in_place_type(T), std::forward<Args>(args)...);
+}
+
+template< class T, class U, class ...Args >
+inline any make_any( std::initializer_list<U> il, Args&& ...args )
+{
+ return any( nstd_lite_in_place_type(T), il, std::forward<Args>(args)...);
+}
+
+#endif // any_CPP11_OR_GREATER
+
+template<
+ class ValueType
+#if any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG
+// any_REQUIRES_T(...) Allow for VC120 (VS2013):
+ , typename = typename std::enable_if< (std::is_reference<ValueType>::value || std::is_copy_constructible<ValueType>::value), nstd::any_lite::detail::enabler >::type
+#endif
+>
+any_nodiscard inline ValueType any_cast( any const & operand )
+{
+ const ValueType * result = any_cast< typename std11::add_const< typename std11::remove_reference<ValueType>::type >::type >( &operand );
+
+#if any_CONFIG_NO_EXCEPTIONS
+ assert( result );
+#else
+ if ( ! result )
+ {
+ throw bad_any_cast();
+ }
+#endif
+
+ return *result;
+}
+
+template<
+ class ValueType
+#if any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG
+// any_REQUIRES_T(...) Allow for VC120 (VS2013):
+ , typename = typename std::enable_if< (std::is_reference<ValueType>::value || std::is_copy_constructible<ValueType>::value), nstd::any_lite::detail::enabler >::type
+#endif
+>
+any_nodiscard inline ValueType any_cast( any & operand )
+{
+ const ValueType * result = any_cast< typename std11::remove_reference<ValueType>::type >( &operand );
+
+#if any_CONFIG_NO_EXCEPTIONS
+ assert( result );
+#else
+ if ( ! result )
+ {
+ throw bad_any_cast();
+ }
+#endif
+
+ return *result;
+}
+
+#if any_CPP11_OR_GREATER
+
+template<
+ class ValueType
+#if any_HAVE_DEFAULT_FUNCTION_TEMPLATE_ARG
+ any_REQUIRES_T( std::is_reference<ValueType>::value || std::is_copy_constructible<ValueType>::value )
+#endif
+>
+any_nodiscard inline ValueType any_cast( any && operand )
+{
+ const ValueType * result = any_cast< typename std11::remove_reference<ValueType>::type >( &operand );
+
+#if any_CONFIG_NO_EXCEPTIONS
+ assert( result );
+#else
+ if ( ! result )
+ {
+ throw bad_any_cast();
+ }
+#endif
+
+ return *result;
+}
+
+#endif // any_CPP11_OR_GREATER
+
+template< class ValueType >
+any_nodiscard inline ValueType const * any_cast( any const * operand ) any_noexcept
+{
+ return operand != any_nullptr && operand->type() == typeid(ValueType) ? operand->to_ptr<ValueType>() : any_nullptr;
+}
+
+template<class ValueType >
+any_nodiscard inline ValueType * any_cast( any * operand ) any_noexcept
+{
+ return operand != any_nullptr && operand->type() == typeid(ValueType) ? operand->to_ptr<ValueType>() : any_nullptr;
+}
+
+} // namespace any_lite
+
+using namespace any_lite;
+
+} // namespace nstd
+
+#endif // any_USES_STD_ANY
+
+#endif // NSTD_ANY_LITE_HPP
--- /dev/null
+//
+// Copyright (c) 2014-2018 Martin Moene
+//
+// https://github.com/martinmoene/optional-lite
+//
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#define optional_lite_MAJOR 3
+#define optional_lite_MINOR 2
+#define optional_lite_PATCH 0
+
+#define optional_lite_VERSION optional_STRINGIFY(optional_lite_MAJOR) "." optional_STRINGIFY(optional_lite_MINOR) "." optional_STRINGIFY(optional_lite_PATCH)
+
+#define optional_STRINGIFY( x ) optional_STRINGIFY_( x )
+#define optional_STRINGIFY_( x ) #x
+
+// optional-lite configuration:
+
+#define optional_OPTIONAL_DEFAULT 0
+#define optional_OPTIONAL_NONSTD 1
+#define optional_OPTIONAL_STD 2
+
+#if !defined( optional_CONFIG_SELECT_OPTIONAL )
+# define optional_CONFIG_SELECT_OPTIONAL ( optional_HAVE_STD_OPTIONAL ? optional_OPTIONAL_STD : optional_OPTIONAL_NONSTD )
+#endif
+
+// Control presence of exception handling (try and auto discover):
+
+#ifndef optional_CONFIG_NO_EXCEPTIONS
+# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+# define optional_CONFIG_NO_EXCEPTIONS 0
+# else
+# define optional_CONFIG_NO_EXCEPTIONS 1
+# endif
+#endif
+
+// C++ language version detection (C++20 is speculative):
+// Note: VC14.0/1900 (VS2015) lacks too much from C++14.
+
+#ifndef optional_CPLUSPLUS
+# if defined(_MSVC_LANG ) && !defined(__clang__)
+# define optional_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG )
+# else
+# define optional_CPLUSPLUS __cplusplus
+# endif
+#endif
+
+#define optional_CPP98_OR_GREATER ( optional_CPLUSPLUS >= 199711L )
+#define optional_CPP11_OR_GREATER ( optional_CPLUSPLUS >= 201103L )
+#define optional_CPP11_OR_GREATER_ ( optional_CPLUSPLUS >= 201103L )
+#define optional_CPP14_OR_GREATER ( optional_CPLUSPLUS >= 201402L )
+#define optional_CPP17_OR_GREATER ( optional_CPLUSPLUS >= 201703L )
+#define optional_CPP20_OR_GREATER ( optional_CPLUSPLUS >= 202000L )
+
+// C++ language version (represent 98 as 3):
+
+#define optional_CPLUSPLUS_V ( optional_CPLUSPLUS / 100 - (optional_CPLUSPLUS > 200000 ? 2000 : 1994) )
+
+// Use C++17 std::optional if available and requested:
+
+#if optional_CPP17_OR_GREATER && defined(__has_include )
+# if __has_include( <optional> )
+# define optional_HAVE_STD_OPTIONAL 1
+# else
+# define optional_HAVE_STD_OPTIONAL 0
+# endif
+#else
+# define optional_HAVE_STD_OPTIONAL 0
+#endif
+
+#define optional_USES_STD_OPTIONAL ( (optional_CONFIG_SELECT_OPTIONAL == optional_OPTIONAL_STD) || ((optional_CONFIG_SELECT_OPTIONAL == optional_OPTIONAL_DEFAULT) && optional_HAVE_STD_OPTIONAL) )
+
+//
+// in_place: code duplicated in any-lite, expected-lite, optional-lite, value-ptr-lite, variant-lite:
+//
+
+#ifndef nonstd_lite_HAVE_IN_PLACE_TYPES
+#define nonstd_lite_HAVE_IN_PLACE_TYPES 1
+
+// C++17 std::in_place in <utility>:
+
+#if optional_CPP17_OR_GREATER
+
+#include <utility>
+
+namespace nonstd {
+
+using std::in_place;
+using std::in_place_type;
+using std::in_place_index;
+using std::in_place_t;
+using std::in_place_type_t;
+using std::in_place_index_t;
+
+#define nonstd_lite_in_place_t( T) std::in_place_t
+#define nonstd_lite_in_place_type_t( T) std::in_place_type_t<T>
+#define nonstd_lite_in_place_index_t(K) std::in_place_index_t<K>
+
+#define nonstd_lite_in_place( T) std::in_place_t{}
+#define nonstd_lite_in_place_type( T) std::in_place_type_t<T>{}
+#define nonstd_lite_in_place_index(K) std::in_place_index_t<K>{}
+
+} // namespace nonstd
+
+#else // optional_CPP17_OR_GREATER
+
+#include <cstddef>
+
+namespace nonstd {
+namespace detail {
+
+template< class T >
+struct in_place_type_tag {};
+
+template< std::size_t K >
+struct in_place_index_tag {};
+
+} // namespace detail
+
+struct in_place_t {};
+
+template< class T >
+inline in_place_t in_place( detail::in_place_type_tag<T> /*unused*/ = detail::in_place_type_tag<T>() )
+{
+ return in_place_t();
+}
+
+template< std::size_t K >
+inline in_place_t in_place( detail::in_place_index_tag<K> /*unused*/ = detail::in_place_index_tag<K>() )
+{
+ return in_place_t();
+}
+
+template< class T >
+inline in_place_t in_place_type( detail::in_place_type_tag<T> /*unused*/ = detail::in_place_type_tag<T>() )
+{
+ return in_place_t();
+}
+
+template< std::size_t K >
+inline in_place_t in_place_index( detail::in_place_index_tag<K> /*unused*/ = detail::in_place_index_tag<K>() )
+{
+ return in_place_t();
+}
+
+// mimic templated typedef:
+
+#define nonstd_lite_in_place_t( T) nonstd::in_place_t(&)( nonstd::detail::in_place_type_tag<T> )
+#define nonstd_lite_in_place_type_t( T) nonstd::in_place_t(&)( nonstd::detail::in_place_type_tag<T> )
+#define nonstd_lite_in_place_index_t(K) nonstd::in_place_t(&)( nonstd::detail::in_place_index_tag<K> )
+
+#define nonstd_lite_in_place( T) nonstd::in_place_type<T>
+#define nonstd_lite_in_place_type( T) nonstd::in_place_type<T>
+#define nonstd_lite_in_place_index(K) nonstd::in_place_index<K>
+
+} // namespace nonstd
+
+#endif // optional_CPP17_OR_GREATER
+#endif // nonstd_lite_HAVE_IN_PLACE_TYPES
+
+//
+// Using std::optional:
+//
+
+#if optional_USES_STD_OPTIONAL
+
+#include <optional>
+
+namespace nonstd {
+
+ using std::optional;
+ using std::bad_optional_access;
+ using std::hash;
+
+ using std::nullopt;
+ using std::nullopt_t;
+
+ using std::operator==;
+ using std::operator!=;
+ using std::operator<;
+ using std::operator<=;
+ using std::operator>;
+ using std::operator>=;
+ using std::make_optional;
+ using std::swap;
+}
+
+#else // optional_USES_STD_OPTIONAL
+
+#include <cassert>
+#include <utility>
+
+// optional-lite alignment configuration:
+
+#ifndef optional_CONFIG_MAX_ALIGN_HACK
+# define optional_CONFIG_MAX_ALIGN_HACK 0
+#endif
+
+#ifndef optional_CONFIG_ALIGN_AS
+// no default, used in #if defined()
+#endif
+
+#ifndef optional_CONFIG_ALIGN_AS_FALLBACK
+# define optional_CONFIG_ALIGN_AS_FALLBACK double
+#endif
+
+// Compiler warning suppression:
+
+#if defined(__clang__)
+# pragma clang diagnostic push
+# pragma clang diagnostic ignored "-Wundef"
+#elif defined(__GNUC__)
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wundef"
+#elif defined(_MSC_VER )
+# pragma warning( push )
+#endif
+
+// half-open range [lo..hi):
+#define optional_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) )
+
+// Compiler versions:
+//
+// MSVC++ 6.0 _MSC_VER == 1200 optional_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0)
+// MSVC++ 7.0 _MSC_VER == 1300 optional_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002)
+// MSVC++ 7.1 _MSC_VER == 1310 optional_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003)
+// MSVC++ 8.0 _MSC_VER == 1400 optional_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005)
+// MSVC++ 9.0 _MSC_VER == 1500 optional_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008)
+// MSVC++ 10.0 _MSC_VER == 1600 optional_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010)
+// MSVC++ 11.0 _MSC_VER == 1700 optional_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012)
+// MSVC++ 12.0 _MSC_VER == 1800 optional_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013)
+// MSVC++ 14.0 _MSC_VER == 1900 optional_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015)
+// MSVC++ 14.1 _MSC_VER >= 1910 optional_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017)
+// MSVC++ 14.2 _MSC_VER >= 1920 optional_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019)
+
+#if defined(_MSC_VER ) && !defined(__clang__)
+# define optional_COMPILER_MSVC_VER (_MSC_VER )
+# define optional_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * ( 5 + (_MSC_VER < 1900 ) ) )
+#else
+# define optional_COMPILER_MSVC_VER 0
+# define optional_COMPILER_MSVC_VERSION 0
+#endif
+
+#define optional_COMPILER_VERSION( major, minor, patch ) ( 10 * (10 * (major) + (minor) ) + (patch) )
+
+#if defined(__GNUC__) && !defined(__clang__)
+# define optional_COMPILER_GNUC_VERSION optional_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
+#else
+# define optional_COMPILER_GNUC_VERSION 0
+#endif
+
+#if defined(__clang__)
+# define optional_COMPILER_CLANG_VERSION optional_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__)
+#else
+# define optional_COMPILER_CLANG_VERSION 0
+#endif
+
+#if optional_BETWEEN(optional_COMPILER_MSVC_VERSION, 70, 140 )
+# pragma warning( disable: 4345 ) // initialization behavior changed
+#endif
+
+#if optional_BETWEEN(optional_COMPILER_MSVC_VERSION, 70, 150 )
+# pragma warning( disable: 4814 ) // in C++14 'constexpr' will not imply 'const'
+#endif
+
+// Presence of language and library features:
+
+#define optional_HAVE(FEATURE) ( optional_HAVE_##FEATURE )
+
+#ifdef _HAS_CPP0X
+# define optional_HAS_CPP0X _HAS_CPP0X
+#else
+# define optional_HAS_CPP0X 0
+#endif
+
+// Unless defined otherwise below, consider VC14 as C++11 for optional-lite:
+
+#if optional_COMPILER_MSVC_VER >= 1900
+# undef optional_CPP11_OR_GREATER
+# define optional_CPP11_OR_GREATER 1
+#endif
+
+#define optional_CPP11_90 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1500)
+#define optional_CPP11_100 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1600)
+#define optional_CPP11_110 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1700)
+#define optional_CPP11_120 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1800)
+#define optional_CPP11_140 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1900)
+#define optional_CPP11_141 (optional_CPP11_OR_GREATER_ || optional_COMPILER_MSVC_VER >= 1910)
+
+#define optional_CPP11_140_490 ((optional_CPP11_OR_GREATER_ && optional_COMPILER_GNUC_VERSION >= 490) || (optional_COMPILER_MSVC_VER >= 1910))
+
+#define optional_CPP14_000 (optional_CPP14_OR_GREATER)
+#define optional_CPP17_000 (optional_CPP17_OR_GREATER)
+
+// Presence of C++11 language features:
+
+#define optional_HAVE_CONSTEXPR_11 optional_CPP11_140
+#define optional_HAVE_IS_DEFAULT optional_CPP11_140
+#define optional_HAVE_NOEXCEPT optional_CPP11_140
+#define optional_HAVE_NULLPTR optional_CPP11_100
+#define optional_HAVE_REF_QUALIFIER optional_CPP11_140_490
+#define optional_HAVE_INITIALIZER_LIST optional_CPP11_140
+
+// Presence of C++14 language features:
+
+#define optional_HAVE_CONSTEXPR_14 optional_CPP14_000
+
+// Presence of C++17 language features:
+
+#define optional_HAVE_NODISCARD optional_CPP17_000
+
+// Presence of C++ library features:
+
+#define optional_HAVE_CONDITIONAL optional_CPP11_120
+#define optional_HAVE_REMOVE_CV optional_CPP11_120
+#define optional_HAVE_TYPE_TRAITS optional_CPP11_90
+
+#define optional_HAVE_TR1_TYPE_TRAITS (!! optional_COMPILER_GNUC_VERSION )
+#define optional_HAVE_TR1_ADD_POINTER (!! optional_COMPILER_GNUC_VERSION )
+
+// C++ feature usage:
+
+#if optional_HAVE( CONSTEXPR_11 )
+# define optional_constexpr constexpr
+#else
+# define optional_constexpr /*constexpr*/
+#endif
+
+#if optional_HAVE( IS_DEFAULT )
+# define optional_is_default = default;
+#else
+# define optional_is_default {}
+#endif
+
+#if optional_HAVE( CONSTEXPR_14 )
+# define optional_constexpr14 constexpr
+#else
+# define optional_constexpr14 /*constexpr*/
+#endif
+
+#if optional_HAVE( NODISCARD )
+# define optional_nodiscard [[nodiscard]]
+#else
+# define optional_nodiscard /*[[nodiscard]]*/
+#endif
+
+#if optional_HAVE( NOEXCEPT )
+# define optional_noexcept noexcept
+#else
+# define optional_noexcept /*noexcept*/
+#endif
+
+#if optional_HAVE( NULLPTR )
+# define optional_nullptr nullptr
+#else
+# define optional_nullptr NULL
+#endif
+
+#if optional_HAVE( REF_QUALIFIER )
+// NOLINTNEXTLINE( bugprone-macro-parentheses )
+# define optional_ref_qual &
+# define optional_refref_qual &&
+#else
+# define optional_ref_qual /*&*/
+# define optional_refref_qual /*&&*/
+#endif
+
+// additional includes:
+
+#if optional_CONFIG_NO_EXCEPTIONS
+// already included: <cassert>
+#else
+# include <stdexcept>
+#endif
+
+#if optional_CPP11_OR_GREATER
+# include <functional>
+#endif
+
+#if optional_HAVE( INITIALIZER_LIST )
+# include <initializer_list>
+#endif
+
+#if optional_HAVE( TYPE_TRAITS )
+# include <type_traits>
+#elif optional_HAVE( TR1_TYPE_TRAITS )
+# include <tr1/type_traits>
+#endif
+
+// Method enabling
+
+#if optional_CPP11_OR_GREATER
+
+#define optional_REQUIRES_0(...) \
+ template< bool B = (__VA_ARGS__), typename std::enable_if<B, int>::type = 0 >
+
+#define optional_REQUIRES_T(...) \
+ , typename std::enable_if< (__VA_ARGS__), int >::type = 0
+
+#define optional_REQUIRES_R(R, ...) \
+ typename std::enable_if< (__VA_ARGS__), R>::type
+
+#define optional_REQUIRES_A(...) \
+ , typename std::enable_if< (__VA_ARGS__), void*>::type = nullptr
+
+#endif
+
+//
+// optional:
+//
+
+namespace nonstd { namespace optional_lite {
+
+namespace std11 {
+
+#if optional_CPP11_OR_GREATER
+ using std::move;
+#else
+ template< typename T > T & move( T & t ) { return t; }
+#endif
+
+#if optional_HAVE( CONDITIONAL )
+ using std::conditional;
+#else
+ template< bool B, typename T, typename F > struct conditional { typedef T type; };
+ template< typename T, typename F > struct conditional<false, T, F> { typedef F type; };
+#endif // optional_HAVE_CONDITIONAL
+
+// gcc < 5:
+#if optional_CPP11_OR_GREATER
+#if optional_BETWEEN( optional_COMPILER_GNUC_VERSION, 1, 500 )
+ template< typename T > struct is_trivially_copy_constructible : std::true_type{};
+ template< typename T > struct is_trivially_move_constructible : std::true_type{};
+#else
+ using std::is_trivially_copy_constructible;
+ using std::is_trivially_move_constructible;
+#endif
+#endif
+} // namespace std11
+
+#if optional_CPP11_OR_GREATER
+
+/// type traits C++17:
+
+namespace std17 {
+
+#if optional_CPP17_OR_GREATER
+
+using std::is_swappable;
+using std::is_nothrow_swappable;
+
+#elif optional_CPP11_OR_GREATER
+
+namespace detail {
+
+using std::swap;
+
+struct is_swappable
+{
+ template< typename T, typename = decltype( swap( std::declval<T&>(), std::declval<T&>() ) ) >
+ static std::true_type test( int /*unused*/ );
+
+ template< typename >
+ static std::false_type test(...);
+};
+
+struct is_nothrow_swappable
+{
+ // wrap noexcept(expr) in separate function as work-around for VC140 (VS2015):
+
+ template< typename T >
+ static constexpr bool satisfies()
+ {
+ return noexcept( swap( std::declval<T&>(), std::declval<T&>() ) );
+ }
+
+ template< typename T >
+ static auto test( int /*unused*/ ) -> std::integral_constant<bool, satisfies<T>()>{}
+
+ template< typename >
+ static auto test(...) -> std::false_type;
+};
+
+} // namespace detail
+
+// is [nothow] swappable:
+
+template< typename T >
+struct is_swappable : decltype( detail::is_swappable::test<T>(0) ){};
+
+template< typename T >
+struct is_nothrow_swappable : decltype( detail::is_nothrow_swappable::test<T>(0) ){};
+
+#endif // optional_CPP17_OR_GREATER
+
+} // namespace std17
+
+/// type traits C++20:
+
+namespace std20 {
+
+template< typename T >
+struct remove_cvref
+{
+ typedef typename std::remove_cv< typename std::remove_reference<T>::type >::type type;
+};
+
+} // namespace std20
+
+#endif // optional_CPP11_OR_GREATER
+
+/// class optional
+
+template< typename T >
+class optional;
+
+namespace detail {
+
+// C++11 emulation:
+
+struct nulltype{};
+
+template< typename Head, typename Tail >
+struct typelist
+{
+ typedef Head head;
+ typedef Tail tail;
+};
+
+#if optional_CONFIG_MAX_ALIGN_HACK
+
+// Max align, use most restricted type for alignment:
+
+#define optional_UNIQUE( name ) optional_UNIQUE2( name, __LINE__ )
+#define optional_UNIQUE2( name, line ) optional_UNIQUE3( name, line )
+#define optional_UNIQUE3( name, line ) name ## line
+
+#define optional_ALIGN_TYPE( type ) \
+ type optional_UNIQUE( _t ); struct_t< type > optional_UNIQUE( _st )
+
+template< typename T >
+struct struct_t { T _; };
+
+union max_align_t
+{
+ optional_ALIGN_TYPE( char );
+ optional_ALIGN_TYPE( short int );
+ optional_ALIGN_TYPE( int );
+ optional_ALIGN_TYPE( long int );
+ optional_ALIGN_TYPE( float );
+ optional_ALIGN_TYPE( double );
+ optional_ALIGN_TYPE( long double );
+ optional_ALIGN_TYPE( char * );
+ optional_ALIGN_TYPE( short int * );
+ optional_ALIGN_TYPE( int * );
+ optional_ALIGN_TYPE( long int * );
+ optional_ALIGN_TYPE( float * );
+ optional_ALIGN_TYPE( double * );
+ optional_ALIGN_TYPE( long double * );
+ optional_ALIGN_TYPE( void * );
+
+#ifdef HAVE_LONG_LONG
+ optional_ALIGN_TYPE( long long );
+#endif
+
+ struct Unknown;
+
+ Unknown ( * optional_UNIQUE(_) )( Unknown );
+ Unknown * Unknown::* optional_UNIQUE(_);
+ Unknown ( Unknown::* optional_UNIQUE(_) )( Unknown );
+
+ struct_t< Unknown ( * )( Unknown) > optional_UNIQUE(_);
+ struct_t< Unknown * Unknown::* > optional_UNIQUE(_);
+ struct_t< Unknown ( Unknown::* )(Unknown) > optional_UNIQUE(_);
+};
+
+#undef optional_UNIQUE
+#undef optional_UNIQUE2
+#undef optional_UNIQUE3
+
+#undef optional_ALIGN_TYPE
+
+#elif defined( optional_CONFIG_ALIGN_AS ) // optional_CONFIG_MAX_ALIGN_HACK
+
+// Use user-specified type for alignment:
+
+#define optional_ALIGN_AS( unused ) \
+ optional_CONFIG_ALIGN_AS
+
+#else // optional_CONFIG_MAX_ALIGN_HACK
+
+// Determine POD type to use for alignment:
+
+#define optional_ALIGN_AS( to_align ) \
+ typename type_of_size< alignment_types, alignment_of< to_align >::value >::type
+
+template< typename T >
+struct alignment_of;
+
+template< typename T >
+struct alignment_of_hack
+{
+ char c;
+ T t;
+ alignment_of_hack();
+};
+
+template< size_t A, size_t S >
+struct alignment_logic
+{
+ enum { value = A < S ? A : S };
+};
+
+template< typename T >
+struct alignment_of
+{
+ enum { value = alignment_logic<
+ sizeof( alignment_of_hack<T> ) - sizeof(T), sizeof(T) >::value };
+};
+
+template< typename List, size_t N >
+struct type_of_size
+{
+ typedef typename std11::conditional<
+ N == sizeof( typename List::head ),
+ typename List::head,
+ typename type_of_size<typename List::tail, N >::type >::type type;
+};
+
+template< size_t N >
+struct type_of_size< nulltype, N >
+{
+ typedef optional_CONFIG_ALIGN_AS_FALLBACK type;
+};
+
+template< typename T>
+struct struct_t { T _; };
+
+#define optional_ALIGN_TYPE( type ) \
+ typelist< type , typelist< struct_t< type >
+
+struct Unknown;
+
+typedef
+ optional_ALIGN_TYPE( char ),
+ optional_ALIGN_TYPE( short ),
+ optional_ALIGN_TYPE( int ),
+ optional_ALIGN_TYPE( long ),
+ optional_ALIGN_TYPE( float ),
+ optional_ALIGN_TYPE( double ),
+ optional_ALIGN_TYPE( long double ),
+
+ optional_ALIGN_TYPE( char *),
+ optional_ALIGN_TYPE( short * ),
+ optional_ALIGN_TYPE( int * ),
+ optional_ALIGN_TYPE( long * ),
+ optional_ALIGN_TYPE( float * ),
+ optional_ALIGN_TYPE( double * ),
+ optional_ALIGN_TYPE( long double * ),
+
+ optional_ALIGN_TYPE( Unknown ( * )( Unknown ) ),
+ optional_ALIGN_TYPE( Unknown * Unknown::* ),
+ optional_ALIGN_TYPE( Unknown ( Unknown::* )( Unknown ) ),
+
+ nulltype
+ > > > > > > > > > > > > > >
+ > > > > > > > > > > > > > >
+ > > > > > >
+ alignment_types;
+
+#undef optional_ALIGN_TYPE
+
+#endif // optional_CONFIG_MAX_ALIGN_HACK
+
+/// C++03 constructed union to hold value.
+
+template< typename T >
+union storage_t
+{
+//private:
+// template< typename > friend class optional;
+
+ typedef T value_type;
+
+ storage_t() optional_is_default
+
+ explicit storage_t( value_type const & v )
+ {
+ construct_value( v );
+ }
+
+ void construct_value( value_type const & v )
+ {
+ ::new( value_ptr() ) value_type( v );
+ }
+
+#if optional_CPP11_OR_GREATER
+
+ explicit storage_t( value_type && v )
+ {
+ construct_value( std::move( v ) );
+ }
+
+ void construct_value( value_type && v )
+ {
+ ::new( value_ptr() ) value_type( std::move( v ) );
+ }
+
+ template< class... Args >
+ void emplace( Args&&... args )
+ {
+ ::new( value_ptr() ) value_type( std::forward<Args>(args)... );
+ }
+
+ template< class U, class... Args >
+ void emplace( std::initializer_list<U> il, Args&&... args )
+ {
+ ::new( value_ptr() ) value_type( il, std::forward<Args>(args)... );
+ }
+
+#endif
+
+ void destruct_value()
+ {
+ value_ptr()->~T();
+ }
+
+ optional_nodiscard value_type const * value_ptr() const
+ {
+ return as<value_type>();
+ }
+
+ value_type * value_ptr()
+ {
+ return as<value_type>();
+ }
+
+ optional_nodiscard value_type const & value() const optional_ref_qual
+ {
+ return * value_ptr();
+ }
+
+ value_type & value() optional_ref_qual
+ {
+ return * value_ptr();
+ }
+
+#if optional_HAVE( REF_QUALIFIER )
+
+ optional_nodiscard value_type const && value() const optional_refref_qual
+ {
+ return std::move( value() );
+ }
+
+ value_type && value() optional_refref_qual
+ {
+ return std::move( value() );
+ }
+
+#endif
+
+#if optional_CPP11_OR_GREATER
+
+ using aligned_storage_t = typename std::aligned_storage< sizeof(value_type), alignof(value_type) >::type;
+ aligned_storage_t data;
+
+#elif optional_CONFIG_MAX_ALIGN_HACK
+
+ typedef struct { unsigned char data[ sizeof(value_type) ]; } aligned_storage_t;
+
+ max_align_t hack;
+ aligned_storage_t data;
+
+#else
+ typedef optional_ALIGN_AS(value_type) align_as_type;
+
+ typedef struct { align_as_type data[ 1 + ( sizeof(value_type) - 1 ) / sizeof(align_as_type) ]; } aligned_storage_t;
+ aligned_storage_t data;
+
+# undef optional_ALIGN_AS
+
+#endif // optional_CONFIG_MAX_ALIGN_HACK
+
+ optional_nodiscard void * ptr() optional_noexcept
+ {
+ return &data;
+ }
+
+ optional_nodiscard void const * ptr() const optional_noexcept
+ {
+ return &data;
+ }
+
+ template <typename U>
+ optional_nodiscard U * as()
+ {
+ return reinterpret_cast<U*>( ptr() );
+ }
+
+ template <typename U>
+ optional_nodiscard U const * as() const
+ {
+ return reinterpret_cast<U const *>( ptr() );
+ }
+};
+
+} // namespace detail
+
+/// disengaged state tag
+
+struct nullopt_t
+{
+ struct init{};
+ explicit optional_constexpr nullopt_t( init /*unused*/ ) optional_noexcept {}
+};
+
+#if optional_HAVE( CONSTEXPR_11 )
+constexpr nullopt_t nullopt{ nullopt_t::init{} };
+#else
+// extra parenthesis to prevent the most vexing parse:
+const nullopt_t nullopt(( nullopt_t::init() ));
+#endif
+
+/// optional access error
+
+#if ! optional_CONFIG_NO_EXCEPTIONS
+
+class bad_optional_access : public std::logic_error
+{
+public:
+ explicit bad_optional_access()
+ : logic_error( "bad optional access" ) {}
+};
+
+#endif //optional_CONFIG_NO_EXCEPTIONS
+
+/// optional
+
+template< typename T>
+class optional
+{
+private:
+ template< typename > friend class optional;
+
+ typedef void (optional::*safe_bool)() const;
+
+public:
+ typedef T value_type;
+
+ // x.x.3.1, constructors
+
+ // 1a - default construct
+ optional_constexpr optional() optional_noexcept
+ : has_value_( false )
+ , contained()
+ {}
+
+ // 1b - construct explicitly empty
+ // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions )
+ optional_constexpr optional( nullopt_t /*unused*/ ) optional_noexcept
+ : has_value_( false )
+ , contained()
+ {}
+
+ // 2 - copy-construct
+#if optional_CPP11_OR_GREATER
+ // template< typename U = T
+ // optional_REQUIRES_T(
+ // std::is_copy_constructible<U>::value
+ // || std11::is_trivially_copy_constructible<U>::value
+ // )
+ // >
+#endif
+ optional_constexpr14 optional( optional const & other )
+ : has_value_( other.has_value() )
+ {
+ if ( other.has_value() )
+ {
+ contained.construct_value( other.contained.value() );
+ }
+ }
+
+#if optional_CPP11_OR_GREATER
+
+ // 3 (C++11) - move-construct from optional
+ template< typename U = T
+ optional_REQUIRES_T(
+ std::is_move_constructible<U>::value
+ || std11::is_trivially_move_constructible<U>::value
+ )
+ >
+ optional_constexpr14 optional( optional && other )
+ // NOLINTNEXTLINE( performance-noexcept-move-constructor )
+ noexcept( std::is_nothrow_move_constructible<T>::value )
+ : has_value_( other.has_value() )
+ {
+ if ( other.has_value() )
+ {
+ contained.construct_value( std::move( other.contained.value() ) );
+ }
+ }
+
+ // 4a (C++11) - explicit converting copy-construct from optional
+ template< typename U
+ optional_REQUIRES_T(
+ std::is_constructible<T, U const &>::value
+ && !std::is_constructible<T, optional<U> & >::value
+ && !std::is_constructible<T, optional<U> && >::value
+ && !std::is_constructible<T, optional<U> const & >::value
+ && !std::is_constructible<T, optional<U> const && >::value
+ && !std::is_convertible< optional<U> & , T>::value
+ && !std::is_convertible< optional<U> && , T>::value
+ && !std::is_convertible< optional<U> const & , T>::value
+ && !std::is_convertible< optional<U> const &&, T>::value
+ && !std::is_convertible< U const & , T>::value /*=> explicit */
+ )
+ >
+ explicit optional( optional<U> const & other )
+ : has_value_( other.has_value() )
+ {
+ if ( other.has_value() )
+ {
+ contained.construct_value( T{ other.contained.value() } );
+ }
+ }
+#endif // optional_CPP11_OR_GREATER
+
+ // 4b (C++98 and later) - non-explicit converting copy-construct from optional
+ template< typename U
+#if optional_CPP11_OR_GREATER
+ optional_REQUIRES_T(
+ std::is_constructible<T, U const &>::value
+ && !std::is_constructible<T, optional<U> & >::value
+ && !std::is_constructible<T, optional<U> && >::value
+ && !std::is_constructible<T, optional<U> const & >::value
+ && !std::is_constructible<T, optional<U> const && >::value
+ && !std::is_convertible< optional<U> & , T>::value
+ && !std::is_convertible< optional<U> && , T>::value
+ && !std::is_convertible< optional<U> const & , T>::value
+ && !std::is_convertible< optional<U> const &&, T>::value
+ && std::is_convertible< U const & , T>::value /*=> non-explicit */
+ )
+#endif // optional_CPP11_OR_GREATER
+ >
+ // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions )
+ /*non-explicit*/ optional( optional<U> const & other )
+ : has_value_( other.has_value() )
+ {
+ if ( other.has_value() )
+ {
+ contained.construct_value( other.contained.value() );
+ }
+ }
+
+#if optional_CPP11_OR_GREATER
+
+ // 5a (C++11) - explicit converting move-construct from optional
+ template< typename U
+ optional_REQUIRES_T(
+ std::is_constructible<T, U &&>::value
+ && !std::is_constructible<T, optional<U> & >::value
+ && !std::is_constructible<T, optional<U> && >::value
+ && !std::is_constructible<T, optional<U> const & >::value
+ && !std::is_constructible<T, optional<U> const && >::value
+ && !std::is_convertible< optional<U> & , T>::value
+ && !std::is_convertible< optional<U> && , T>::value
+ && !std::is_convertible< optional<U> const & , T>::value
+ && !std::is_convertible< optional<U> const &&, T>::value
+ && !std::is_convertible< U &&, T>::value /*=> explicit */
+ )
+ >
+ explicit optional( optional<U> && other
+ )
+ : has_value_( other.has_value() )
+ {
+ if ( other.has_value() )
+ {
+ contained.construct_value( T{ std::move( other.contained.value() ) } );
+ }
+ }
+
+ // 5a (C++11) - non-explicit converting move-construct from optional
+ template< typename U
+ optional_REQUIRES_T(
+ std::is_constructible<T, U &&>::value
+ && !std::is_constructible<T, optional<U> & >::value
+ && !std::is_constructible<T, optional<U> && >::value
+ && !std::is_constructible<T, optional<U> const & >::value
+ && !std::is_constructible<T, optional<U> const && >::value
+ && !std::is_convertible< optional<U> & , T>::value
+ && !std::is_convertible< optional<U> && , T>::value
+ && !std::is_convertible< optional<U> const & , T>::value
+ && !std::is_convertible< optional<U> const &&, T>::value
+ && std::is_convertible< U &&, T>::value /*=> non-explicit */
+ )
+ >
+ // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions )
+ /*non-explicit*/ optional( optional<U> && other )
+ : has_value_( other.has_value() )
+ {
+ if ( other.has_value() )
+ {
+ contained.construct_value( std::move( other.contained.value() ) );
+ }
+ }
+
+ // 6 (C++11) - in-place construct
+ template< typename... Args
+ optional_REQUIRES_T(
+ std::is_constructible<T, Args&&...>::value
+ )
+ >
+ optional_constexpr explicit optional( nonstd_lite_in_place_t(T), Args&&... args )
+ : has_value_( true )
+ , contained( T( std::forward<Args>(args)...) )
+ {}
+
+ // 7 (C++11) - in-place construct, initializer-list
+ template< typename U, typename... Args
+ optional_REQUIRES_T(
+ std::is_constructible<T, std::initializer_list<U>&, Args&&...>::value
+ )
+ >
+ optional_constexpr explicit optional( nonstd_lite_in_place_t(T), std::initializer_list<U> il, Args&&... args )
+ : has_value_( true )
+ , contained( T( il, std::forward<Args>(args)...) )
+ {}
+
+ // 8a (C++11) - explicit move construct from value
+ template< typename U = T
+ optional_REQUIRES_T(
+ std::is_constructible<T, U&&>::value
+ && !std::is_same<typename std20::remove_cvref<U>::type, nonstd_lite_in_place_t(U)>::value
+ && !std::is_same<typename std20::remove_cvref<U>::type, optional<T>>::value
+ && !std::is_convertible<U&&, T>::value /*=> explicit */
+ )
+ >
+ optional_constexpr explicit optional( U && value )
+ : has_value_( true )
+ , contained( T{ std::forward<U>( value ) } )
+ {}
+
+ // 8b (C++11) - non-explicit move construct from value
+ template< typename U = T
+ optional_REQUIRES_T(
+ std::is_constructible<T, U&&>::value
+ && !std::is_same<typename std20::remove_cvref<U>::type, nonstd_lite_in_place_t(U)>::value
+ && !std::is_same<typename std20::remove_cvref<U>::type, optional<T>>::value
+ && std::is_convertible<U&&, T>::value /*=> non-explicit */
+ )
+ >
+ // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions )
+ optional_constexpr /*non-explicit*/ optional( U && value )
+ : has_value_( true )
+ , contained( std::forward<U>( value ) )
+ {}
+
+#else // optional_CPP11_OR_GREATER
+
+ // 8 (C++98)
+ optional( value_type const & value )
+ : has_value_( true )
+ , contained( value )
+ {}
+
+#endif // optional_CPP11_OR_GREATER
+
+ // x.x.3.2, destructor
+
+ ~optional()
+ {
+ if ( has_value() )
+ {
+ contained.destruct_value();
+ }
+ }
+
+ // x.x.3.3, assignment
+
+ // 1 (C++98and later) - assign explicitly empty
+ optional & operator=( nullopt_t /*unused*/) optional_noexcept
+ {
+ reset();
+ return *this;
+ }
+
+ // 2 (C++98and later) - copy-assign from optional
+#if optional_CPP11_OR_GREATER
+ // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator )
+ optional_REQUIRES_R(
+ optional &,
+ true
+// std::is_copy_constructible<T>::value
+// && std::is_copy_assignable<T>::value
+ )
+ operator=( optional const & other )
+ noexcept(
+ std::is_nothrow_move_assignable<T>::value
+ && std::is_nothrow_move_constructible<T>::value
+ )
+#else
+ optional & operator=( optional const & other )
+#endif
+ {
+ if ( (has_value() == true ) && (other.has_value() == false) ) { reset(); }
+ else if ( (has_value() == false) && (other.has_value() == true ) ) { initialize( *other ); }
+ else if ( (has_value() == true ) && (other.has_value() == true ) ) { contained.value() = *other; }
+ return *this;
+ }
+
+#if optional_CPP11_OR_GREATER
+
+ // 3 (C++11) - move-assign from optional
+ // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator )
+ optional_REQUIRES_R(
+ optional &,
+ true
+// std::is_move_constructible<T>::value
+// && std::is_move_assignable<T>::value
+ )
+ operator=( optional && other ) noexcept
+ {
+ if ( (has_value() == true ) && (other.has_value() == false) ) { reset(); }
+ else if ( (has_value() == false) && (other.has_value() == true ) ) { initialize( std::move( *other ) ); }
+ else if ( (has_value() == true ) && (other.has_value() == true ) ) { contained.value() = std::move( *other ); }
+ return *this;
+ }
+
+ // 4 (C++11) - move-assign from value
+ template< typename U = T >
+ // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator )
+ optional_REQUIRES_R(
+ optional &,
+ std::is_constructible<T , U>::value
+ && std::is_assignable<T&, U>::value
+ && !std::is_same<typename std20::remove_cvref<U>::type, nonstd_lite_in_place_t(U)>::value
+ && !std::is_same<typename std20::remove_cvref<U>::type, optional<T>>::value
+ && !(std::is_scalar<T>::value && std::is_same<T, typename std::decay<U>::type>::value)
+ )
+ operator=( U && value )
+ {
+ if ( has_value() )
+ {
+ contained.value() = std::forward<U>( value );
+ }
+ else
+ {
+ initialize( T( std::forward<U>( value ) ) );
+ }
+ return *this;
+ }
+
+#else // optional_CPP11_OR_GREATER
+
+ // 4 (C++98) - copy-assign from value
+ template< typename U /*= T*/ >
+ optional & operator=( U const & value )
+ {
+ if ( has_value() ) contained.value() = value;
+ else initialize( T( value ) );
+ return *this;
+ }
+
+#endif // optional_CPP11_OR_GREATER
+
+ // 5 (C++98 and later) - converting copy-assign from optional
+ template< typename U >
+#if optional_CPP11_OR_GREATER
+ // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator )
+ optional_REQUIRES_R(
+ optional&,
+ std::is_constructible< T , U const &>::value
+ && std::is_assignable< T&, U const &>::value
+ && !std::is_constructible<T, optional<U> & >::value
+ && !std::is_constructible<T, optional<U> && >::value
+ && !std::is_constructible<T, optional<U> const & >::value
+ && !std::is_constructible<T, optional<U> const && >::value
+ && !std::is_convertible< optional<U> & , T>::value
+ && !std::is_convertible< optional<U> && , T>::value
+ && !std::is_convertible< optional<U> const & , T>::value
+ && !std::is_convertible< optional<U> const &&, T>::value
+ && !std::is_assignable< T&, optional<U> & >::value
+ && !std::is_assignable< T&, optional<U> && >::value
+ && !std::is_assignable< T&, optional<U> const & >::value
+ && !std::is_assignable< T&, optional<U> const && >::value
+ )
+#else
+ optional&
+#endif // optional_CPP11_OR_GREATER
+ operator=( optional<U> const & other )
+ {
+ return *this = optional( other );
+ }
+
+#if optional_CPP11_OR_GREATER
+
+ // 6 (C++11) - converting move-assign from optional
+ template< typename U >
+ // NOLINTNEXTLINE( cppcoreguidelines-c-copy-assignment-signature, misc-unconventional-assign-operator )
+ optional_REQUIRES_R(
+ optional&,
+ std::is_constructible< T , U>::value
+ && std::is_assignable< T&, U>::value
+ && !std::is_constructible<T, optional<U> & >::value
+ && !std::is_constructible<T, optional<U> && >::value
+ && !std::is_constructible<T, optional<U> const & >::value
+ && !std::is_constructible<T, optional<U> const && >::value
+ && !std::is_convertible< optional<U> & , T>::value
+ && !std::is_convertible< optional<U> && , T>::value
+ && !std::is_convertible< optional<U> const & , T>::value
+ && !std::is_convertible< optional<U> const &&, T>::value
+ && !std::is_assignable< T&, optional<U> & >::value
+ && !std::is_assignable< T&, optional<U> && >::value
+ && !std::is_assignable< T&, optional<U> const & >::value
+ && !std::is_assignable< T&, optional<U> const && >::value
+ )
+ operator=( optional<U> && other )
+ {
+ return *this = optional( std::move( other ) );
+ }
+
+ // 7 (C++11) - emplace
+ template< typename... Args
+ optional_REQUIRES_T(
+ std::is_constructible<T, Args&&...>::value
+ )
+ >
+ T& emplace( Args&&... args )
+ {
+ *this = nullopt;
+ contained.emplace( std::forward<Args>(args)... );
+ has_value_ = true;
+ return contained.value();
+ }
+
+ // 8 (C++11) - emplace, initializer-list
+ template< typename U, typename... Args
+ optional_REQUIRES_T(
+ std::is_constructible<T, std::initializer_list<U>&, Args&&...>::value
+ )
+ >
+ T& emplace( std::initializer_list<U> il, Args&&... args )
+ {
+ *this = nullopt;
+ contained.emplace( il, std::forward<Args>(args)... );
+ has_value_ = true;
+ return contained.value();
+ }
+
+#endif // optional_CPP11_OR_GREATER
+
+ // x.x.3.4, swap
+
+ void swap( optional & other )
+#if optional_CPP11_OR_GREATER
+ noexcept(
+ std::is_nothrow_move_constructible<T>::value
+ && std17::is_nothrow_swappable<T>::value
+ )
+#endif
+ {
+ using std::swap;
+ if ( (has_value() == true ) && (other.has_value() == true ) ) { swap( **this, *other ); }
+ else if ( (has_value() == false) && (other.has_value() == true ) ) { initialize( std11::move(*other) ); other.reset(); }
+ else if ( (has_value() == true ) && (other.has_value() == false) ) { other.initialize( std11::move(**this) ); reset(); }
+ }
+
+ // x.x.3.5, observers
+
+ optional_constexpr value_type const * operator ->() const
+ {
+ return assert( has_value() ),
+ contained.value_ptr();
+ }
+
+ optional_constexpr14 value_type * operator ->()
+ {
+ return assert( has_value() ),
+ contained.value_ptr();
+ }
+
+ optional_constexpr value_type const & operator *() const optional_ref_qual
+ {
+ return assert( has_value() ),
+ contained.value();
+ }
+
+ optional_constexpr14 value_type & operator *() optional_ref_qual
+ {
+ return assert( has_value() ),
+ contained.value();
+ }
+
+#if optional_HAVE( REF_QUALIFIER )
+
+ optional_constexpr value_type const && operator *() const optional_refref_qual
+ {
+ return std::move( **this );
+ }
+
+ optional_constexpr14 value_type && operator *() optional_refref_qual
+ {
+ return std::move( **this );
+ }
+
+#endif
+
+#if optional_CPP11_OR_GREATER
+ optional_constexpr explicit operator bool() const optional_noexcept
+ {
+ return has_value();
+ }
+#else
+ optional_constexpr operator safe_bool() const optional_noexcept
+ {
+ return has_value() ? &optional::this_type_does_not_support_comparisons : 0;
+ }
+#endif
+
+ // NOLINTNEXTLINE( modernize-use-nodiscard )
+ /*optional_nodiscard*/ optional_constexpr bool has_value() const optional_noexcept
+ {
+ return has_value_;
+ }
+
+ // NOLINTNEXTLINE( modernize-use-nodiscard )
+ /*optional_nodiscard*/ optional_constexpr14 value_type const & value() const optional_ref_qual
+ {
+#if optional_CONFIG_NO_EXCEPTIONS
+ assert( has_value() );
+#else
+ if ( ! has_value() )
+ {
+ throw bad_optional_access();
+ }
+#endif
+ return contained.value();
+ }
+
+ optional_constexpr14 value_type & value() optional_ref_qual
+ {
+#if optional_CONFIG_NO_EXCEPTIONS
+ assert( has_value() );
+#else
+ if ( ! has_value() )
+ {
+ throw bad_optional_access();
+ }
+#endif
+ return contained.value();
+ }
+
+#if optional_HAVE( REF_QUALIFIER ) && ( !optional_COMPILER_GNUC_VERSION || optional_COMPILER_GNUC_VERSION >= 490 )
+
+ // NOLINTNEXTLINE( modernize-use-nodiscard )
+ /*optional_nodiscard*/ optional_constexpr value_type const && value() const optional_refref_qual
+ {
+ return std::move( value() );
+ }
+
+ optional_constexpr14 value_type && value() optional_refref_qual
+ {
+ return std::move( value() );
+ }
+
+#endif
+
+#if optional_CPP11_OR_GREATER
+
+ template< typename U >
+ optional_constexpr value_type value_or( U && v ) const optional_ref_qual
+ {
+ return has_value() ? contained.value() : static_cast<T>(std::forward<U>( v ) );
+ }
+
+ template< typename U >
+ optional_constexpr14 value_type value_or( U && v ) optional_refref_qual
+ {
+ return has_value() ? std::move( contained.value() ) : static_cast<T>(std::forward<U>( v ) );
+ }
+
+#else
+
+ template< typename U >
+ optional_constexpr value_type value_or( U const & v ) const
+ {
+ return has_value() ? contained.value() : static_cast<value_type>( v );
+ }
+
+#endif // optional_CPP11_OR_GREATER
+
+ // x.x.3.6, modifiers
+
+ void reset() optional_noexcept
+ {
+ if ( has_value() )
+ {
+ contained.destruct_value();
+ }
+
+ has_value_ = false;
+ }
+
+private:
+ void this_type_does_not_support_comparisons() const {}
+
+ template< typename V >
+ void initialize( V const & value )
+ {
+ assert( ! has_value() );
+ contained.construct_value( value );
+ has_value_ = true;
+ }
+
+#if optional_CPP11_OR_GREATER
+ template< typename V >
+ void initialize( V && value )
+ {
+ assert( ! has_value() );
+ contained.construct_value( std::move( value ) );
+ has_value_ = true;
+ }
+
+#endif
+
+private:
+ bool has_value_;
+ detail::storage_t< value_type > contained;
+
+};
+
+// Relational operators
+
+template< typename T, typename U >
+inline optional_constexpr bool operator==( optional<T> const & x, optional<U> const & y )
+{
+ return bool(x) != bool(y) ? false : !bool( x ) ? true : *x == *y;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator!=( optional<T> const & x, optional<U> const & y )
+{
+ return !(x == y);
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator<( optional<T> const & x, optional<U> const & y )
+{
+ return (!y) ? false : (!x) ? true : *x < *y;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator>( optional<T> const & x, optional<U> const & y )
+{
+ return (y < x);
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator<=( optional<T> const & x, optional<U> const & y )
+{
+ return !(y < x);
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator>=( optional<T> const & x, optional<U> const & y )
+{
+ return !(x < y);
+}
+
+// Comparison with nullopt
+
+template< typename T >
+inline optional_constexpr bool operator==( optional<T> const & x, nullopt_t /*unused*/ ) optional_noexcept
+{
+ return (!x);
+}
+
+template< typename T >
+inline optional_constexpr bool operator==( nullopt_t /*unused*/, optional<T> const & x ) optional_noexcept
+{
+ return (!x);
+}
+
+template< typename T >
+inline optional_constexpr bool operator!=( optional<T> const & x, nullopt_t /*unused*/ ) optional_noexcept
+{
+ return bool(x);
+}
+
+template< typename T >
+inline optional_constexpr bool operator!=( nullopt_t /*unused*/, optional<T> const & x ) optional_noexcept
+{
+ return bool(x);
+}
+
+template< typename T >
+inline optional_constexpr bool operator<( optional<T> const & /*unused*/, nullopt_t /*unused*/ ) optional_noexcept
+{
+ return false;
+}
+
+template< typename T >
+inline optional_constexpr bool operator<( nullopt_t /*unused*/, optional<T> const & x ) optional_noexcept
+{
+ return bool(x);
+}
+
+template< typename T >
+inline optional_constexpr bool operator<=( optional<T> const & x, nullopt_t /*unused*/ ) optional_noexcept
+{
+ return (!x);
+}
+
+template< typename T >
+inline optional_constexpr bool operator<=( nullopt_t /*unused*/, optional<T> const & /*unused*/ ) optional_noexcept
+{
+ return true;
+}
+
+template< typename T >
+inline optional_constexpr bool operator>( optional<T> const & x, nullopt_t /*unused*/ ) optional_noexcept
+{
+ return bool(x);
+}
+
+template< typename T >
+inline optional_constexpr bool operator>( nullopt_t /*unused*/, optional<T> const & /*unused*/ ) optional_noexcept
+{
+ return false;
+}
+
+template< typename T >
+inline optional_constexpr bool operator>=( optional<T> const & /*unused*/, nullopt_t /*unused*/ ) optional_noexcept
+{
+ return true;
+}
+
+template< typename T >
+inline optional_constexpr bool operator>=( nullopt_t /*unused*/, optional<T> const & x ) optional_noexcept
+{
+ return (!x);
+}
+
+// Comparison with T
+
+template< typename T, typename U >
+inline optional_constexpr bool operator==( optional<T> const & x, U const & v )
+{
+ return bool(x) ? *x == v : false;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator==( U const & v, optional<T> const & x )
+{
+ return bool(x) ? v == *x : false;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator!=( optional<T> const & x, U const & v )
+{
+ return bool(x) ? *x != v : true;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator!=( U const & v, optional<T> const & x )
+{
+ return bool(x) ? v != *x : true;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator<( optional<T> const & x, U const & v )
+{
+ return bool(x) ? *x < v : true;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator<( U const & v, optional<T> const & x )
+{
+ return bool(x) ? v < *x : false;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator<=( optional<T> const & x, U const & v )
+{
+ return bool(x) ? *x <= v : true;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator<=( U const & v, optional<T> const & x )
+{
+ return bool(x) ? v <= *x : false;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator>( optional<T> const & x, U const & v )
+{
+ return bool(x) ? *x > v : false;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator>( U const & v, optional<T> const & x )
+{
+ return bool(x) ? v > *x : true;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator>=( optional<T> const & x, U const & v )
+{
+ return bool(x) ? *x >= v : false;
+}
+
+template< typename T, typename U >
+inline optional_constexpr bool operator>=( U const & v, optional<T> const & x )
+{
+ return bool(x) ? v >= *x : true;
+}
+
+// Specialized algorithms
+
+template< typename T
+#if optional_CPP11_OR_GREATER
+ optional_REQUIRES_T(
+ std::is_move_constructible<T>::value
+ && std17::is_swappable<T>::value )
+#endif
+>
+void swap( optional<T> & x, optional<T> & y )
+#if optional_CPP11_OR_GREATER
+ noexcept( noexcept( x.swap(y) ) )
+#endif
+{
+ x.swap( y );
+}
+
+#if optional_CPP11_OR_GREATER
+
+template< typename T >
+optional_constexpr optional< typename std::decay<T>::type > make_optional( T && value )
+{
+ return optional< typename std::decay<T>::type >( std::forward<T>( value ) );
+}
+
+template< typename T, typename...Args >
+optional_constexpr optional<T> make_optional( Args&&... args )
+{
+ return optional<T>( nonstd_lite_in_place(T), std::forward<Args>(args)...);
+}
+
+template< typename T, typename U, typename... Args >
+optional_constexpr optional<T> make_optional( std::initializer_list<U> il, Args&&... args )
+{
+ return optional<T>( nonstd_lite_in_place(T), il, std::forward<Args>(args)...);
+}
+
+#else
+
+template< typename T >
+optional<T> make_optional( T const & value )
+{
+ return optional<T>( value );
+}
+
+#endif // optional_CPP11_OR_GREATER
+
+} // namespace optional_lite
+
+using optional_lite::optional;
+using optional_lite::nullopt_t;
+using optional_lite::nullopt;
+
+#if ! optional_CONFIG_NO_EXCEPTIONS
+using optional_lite::bad_optional_access;
+#endif
+
+using optional_lite::make_optional;
+
+} // namespace nonstd
+
+#if optional_CPP11_OR_GREATER
+
+// specialize the std::hash algorithm:
+
+namespace std {
+
+template< class T >
+struct hash< nonstd::optional<T> >
+{
+public:
+ std::size_t operator()( nonstd::optional<T> const & v ) const optional_noexcept
+ {
+ return bool( v ) ? std::hash<T>{}( *v ) : 0;
+ }
+};
+
+} //namespace std
+
+#endif // optional_CPP11_OR_GREATER
+
+#if defined(__clang__)
+# pragma clang diagnostic pop
+#elif defined(__GNUC__)
+# pragma GCC diagnostic pop
+#elif defined(_MSC_VER )
+# pragma warning( pop )
+#endif
+
+#endif // optional_USES_STD_OPTIONAL
+
+
+namespace tf { namespace nstd {
+
+template <typename T>
+using optional = nonstd::optional<T>;
+
+}}
--- /dev/null
+// The file is modified from MPark in order to use variant in C++11/14
+// We change the namespace to stick with the v1.4, in order to avoid clashing
+// problem when upstream code uses the same library.
+
+// MPark.Variant
+//
+// Copyright Michael Park, 2015-2017
+//
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <cstddef>
+#include <exception>
+#include <functional>
+#include <initializer_list>
+#include <limits>
+#include <new>
+#include <type_traits>
+#include <utility>
+#include <memory>
+
+
+// MPark.Variant
+//
+// Copyright Michael Park, 2015-2017
+//
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
+
+// MSVC 2015 Update 3.
+#if __cplusplus < 201103L && (!defined(_MSC_VER) || _MSC_FULL_VER < 190024210)
+#error "MPark.Variant requires C++11 support."
+#endif
+
+#ifndef __has_attribute
+#define __has_attribute(x) 0
+#endif
+
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+#ifndef __has_include
+#define __has_include(x) 0
+#endif
+
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+
+#if __has_attribute(always_inline) || defined(__GNUC__)
+#define TF_ALWAYS_INLINE __attribute__((__always_inline__)) inline
+#elif defined(_MSC_VER)
+#define TF_ALWAYS_INLINE __forceinline
+#else
+#define TF_ALWAYS_INLINE inline
+#endif
+
+#if __has_builtin(__builtin_addressof) || \
+ (defined(__GNUC__) && __GNUC__ >= 7) || defined(_MSC_VER)
+#define TF_BUILTIN_ADDRESSOF
+#endif
+
+#if __has_builtin(__builtin_unreachable) || defined(__GNUC__)
+#define TF_BUILTIN_UNREACHABLE __builtin_unreachable()
+#elif defined(_MSC_VER)
+#define TF_BUILTIN_UNREACHABLE __assume(false)
+#else
+#define TF_BUILTIN_UNREACHABLE
+#endif
+
+#if __has_builtin(__type_pack_element)
+#define TF_TYPE_PACK_ELEMENT
+#endif
+
+#if defined(__cpp_constexpr) && __cpp_constexpr >= 200704 && \
+ !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 9)
+#define TF_CPP11_CONSTEXPR
+#endif
+
+#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304
+#define TF_CPP14_CONSTEXPR
+#endif
+
+#if __has_feature(cxx_exceptions) || defined(__cpp_exceptions) || \
+ (defined(_MSC_VER) && defined(_CPPUNWIND))
+#define TF_EXCEPTIONS
+#endif
+
+#if defined(__cpp_generic_lambdas) || defined(_MSC_VER)
+#define TF_GENERIC_LAMBDAS
+#endif
+
+#if defined(__cpp_lib_integer_sequence)
+#define TF_INTEGER_SEQUENCE
+#endif
+
+#if (defined(__cpp_decltype_auto) && defined(__cpp_return_type_deduction)) || defined(_MSC_VER)
+#define TF_RETURN_TYPE_DEDUCTION
+#endif
+
+#if defined(__cpp_lib_transparent_operators) || defined(_MSC_VER)
+#define TF_TRANSPARENT_OPERATORS
+#endif
+
+#if defined(__cpp_variable_templates) || defined(_MSC_VER)
+#define TF_VARIABLE_TEMPLATES
+#endif
+
+#if !defined(__GLIBCXX__) || __has_include(<codecvt>) // >= libstdc++-5
+#define TF_TRIVIALITY_TYPE_TRAITS
+#define TF_INCOMPLETE_TYPE_TRAITS
+#endif
+
+// MPark.Variant
+//
+// Copyright Michael Park, 2015-2017
+//
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
+
+// in_place_t definition
+namespace tf { namespace nstd {
+
+ struct in_place_t { explicit in_place_t() = default; };
+
+ template <std::size_t I>
+ struct in_place_index_t { explicit in_place_index_t() = default; };
+
+ template <typename T>
+ struct in_place_type_t { explicit in_place_type_t() = default; };
+
+#ifdef TF_VARIABLE_TEMPLATES
+ constexpr in_place_t in_place{};
+
+ template <std::size_t I> constexpr in_place_index_t<I> in_place_index{};
+
+ template <typename T> constexpr in_place_type_t<T> in_place_type{};
+#endif
+
+}} // namespace tf::nstd
+
+// MPark.Variant
+//
+// Copyright Michael Park, 2015-2017
+//
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
+
+
+#define TF_RETURN(...) \
+ noexcept(noexcept(__VA_ARGS__)) -> decltype(__VA_ARGS__) { return __VA_ARGS__; }
+
+namespace tf { namespace nstd {
+ namespace lib {
+ template <typename T>
+ struct identity { using type = T; };
+
+ inline namespace cpp14 {
+ template <typename T, std::size_t N>
+ struct array {
+ constexpr const T &operator[](std::size_t index) const {
+ return data[index];
+ }
+
+ T data[N == 0 ? 1 : N];
+ };
+
+ template <typename T>
+ using add_pointer_t = typename std::add_pointer<T>::type;
+
+ template <typename... Ts>
+ using common_type_t = typename std::common_type<Ts...>::type;
+
+ template <typename T>
+ using decay_t = typename std::decay<T>::type;
+
+ template <bool B, typename T = void>
+ using enable_if_t = typename std::enable_if<B, T>::type;
+
+ template <typename T>
+ using remove_const_t = typename std::remove_const<T>::type;
+
+ template <typename T>
+ using remove_reference_t = typename std::remove_reference<T>::type;
+
+ template <typename T>
+ using remove_cvref_t =
+ typename std::remove_cv<remove_reference_t<T>>::type;
+
+ template <typename T>
+ inline constexpr T &&forward(remove_reference_t<T> &t) noexcept {
+ return static_cast<T &&>(t);
+ }
+
+ template <typename T>
+ inline constexpr T &&forward(remove_reference_t<T> &&t) noexcept {
+ static_assert(!std::is_lvalue_reference<T>::value,
+ "can not forward an rvalue as an lvalue");
+ return static_cast<T &&>(t);
+ }
+
+ template <typename T>
+ inline constexpr remove_reference_t<T> &&move(T &&t) noexcept {
+ return static_cast<remove_reference_t<T> &&>(t);
+ }
+
+#ifdef TF_INTEGER_SEQUENCE
+ using std::integer_sequence;
+ using std::index_sequence;
+ using std::make_index_sequence;
+ using std::index_sequence_for;
+#else
+ template <typename T, T... Is>
+ struct integer_sequence {
+ using value_type = T;
+ static constexpr std::size_t size() noexcept { return sizeof...(Is); }
+ };
+
+ template <std::size_t... Is>
+ using index_sequence = integer_sequence<std::size_t, Is...>;
+
+ template <typename Lhs, typename Rhs>
+ struct make_index_sequence_concat;
+
+ template <std::size_t... Lhs, std::size_t... Rhs>
+ struct make_index_sequence_concat<index_sequence<Lhs...>,
+ index_sequence<Rhs...>>
+ : identity<index_sequence<Lhs..., (sizeof...(Lhs) + Rhs)...>> {};
+
+ template <std::size_t N>
+ struct make_index_sequence_impl;
+
+ template <std::size_t N>
+ using make_index_sequence = typename make_index_sequence_impl<N>::type;
+
+ template <std::size_t N>
+ struct make_index_sequence_impl
+ : make_index_sequence_concat<make_index_sequence<N / 2>,
+ make_index_sequence<N - (N / 2)>> {};
+
+ template <>
+ struct make_index_sequence_impl<0> : identity<index_sequence<>> {};
+
+ template <>
+ struct make_index_sequence_impl<1> : identity<index_sequence<0>> {};
+
+ template <typename... Ts>
+ using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+#endif
+
+ // <functional>
+#ifdef TF_TRANSPARENT_OPERATORS
+ using equal_to = std::equal_to<>;
+#else
+ struct equal_to {
+ template <typename Lhs, typename Rhs>
+ inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const
+ TF_RETURN(lib::forward<Lhs>(lhs) == lib::forward<Rhs>(rhs))
+ };
+#endif
+
+#ifdef TF_TRANSPARENT_OPERATORS
+ using not_equal_to = std::not_equal_to<>;
+#else
+ struct not_equal_to {
+ template <typename Lhs, typename Rhs>
+ inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const
+ TF_RETURN(lib::forward<Lhs>(lhs) != lib::forward<Rhs>(rhs))
+ };
+#endif
+
+#ifdef TF_TRANSPARENT_OPERATORS
+ using less = std::less<>;
+#else
+ struct less {
+ template <typename Lhs, typename Rhs>
+ inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const
+ TF_RETURN(lib::forward<Lhs>(lhs) < lib::forward<Rhs>(rhs))
+ };
+#endif
+
+#ifdef TF_TRANSPARENT_OPERATORS
+ using greater = std::greater<>;
+#else
+ struct greater {
+ template <typename Lhs, typename Rhs>
+ inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const
+ TF_RETURN(lib::forward<Lhs>(lhs) > lib::forward<Rhs>(rhs))
+ };
+#endif
+
+#ifdef TF_TRANSPARENT_OPERATORS
+ using less_equal = std::less_equal<>;
+#else
+ struct less_equal {
+ template <typename Lhs, typename Rhs>
+ inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const
+ TF_RETURN(lib::forward<Lhs>(lhs) <= lib::forward<Rhs>(rhs))
+ };
+#endif
+
+#ifdef TF_TRANSPARENT_OPERATORS
+ using greater_equal = std::greater_equal<>;
+#else
+ struct greater_equal {
+ template <typename Lhs, typename Rhs>
+ inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const
+ TF_RETURN(lib::forward<Lhs>(lhs) >= lib::forward<Rhs>(rhs))
+ };
+#endif
+ } // namespace cpp14
+
+ inline namespace cpp17 {
+
+ // <type_traits>
+ template <bool B>
+ using bool_constant = std::integral_constant<bool, B>;
+
+ template <typename...>
+ struct voider : identity<void> {};
+
+ template <typename... Ts>
+ using void_t = typename voider<Ts...>::type;
+
+ namespace detail {
+ namespace swappable {
+
+ using std::swap;
+
+ template <typename T>
+ struct is_swappable {
+ private:
+ template <typename U,
+ typename = decltype(swap(std::declval<U &>(),
+ std::declval<U &>()))>
+ inline static std::true_type test(int);
+
+ template <typename U>
+ inline static std::false_type test(...);
+
+ public:
+ static constexpr bool value = decltype(test<T>(0))::value;
+ };
+
+ template <bool IsSwappable, typename T>
+ struct is_nothrow_swappable {
+ static constexpr bool value =
+ noexcept(swap(std::declval<T &>(), std::declval<T &>()));
+ };
+
+ template <typename T>
+ struct is_nothrow_swappable<false, T> : std::false_type {};
+
+ } // namespace swappable
+ } // namespace detail
+
+ using detail::swappable::is_swappable;
+
+ template <typename T>
+ using is_nothrow_swappable =
+ detail::swappable::is_nothrow_swappable<is_swappable<T>::value, T>;
+
+ // <functional>
+ namespace detail {
+
+ template <typename T>
+ struct is_reference_wrapper : std::false_type {};
+
+ template <typename T>
+ struct is_reference_wrapper<std::reference_wrapper<T>>
+ : std::true_type {};
+
+ template <bool, int>
+ struct Invoke;
+
+ template <>
+ struct Invoke<true /* pmf */, 0 /* is_base_of */> {
+ template <typename R, typename T, typename Arg, typename... Args>
+ inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args)
+ TF_RETURN((lib::forward<Arg>(arg).*pmf)(lib::forward<Args>(args)...))
+ };
+
+ template <>
+ struct Invoke<true /* pmf */, 1 /* is_reference_wrapper */> {
+ template <typename R, typename T, typename Arg, typename... Args>
+ inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args)
+ TF_RETURN((lib::forward<Arg>(arg).get().*pmf)(lib::forward<Args>(args)...))
+ };
+
+ template <>
+ struct Invoke<true /* pmf */, 2 /* otherwise */> {
+ template <typename R, typename T, typename Arg, typename... Args>
+ inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args)
+ TF_RETURN(((*lib::forward<Arg>(arg)).*pmf)(lib::forward<Args>(args)...))
+ };
+
+ template <>
+ struct Invoke<false /* pmo */, 0 /* is_base_of */> {
+ template <typename R, typename T, typename Arg>
+ inline static constexpr auto invoke(R T::*pmo, Arg &&arg)
+ TF_RETURN(lib::forward<Arg>(arg).*pmo)
+ };
+
+ template <>
+ struct Invoke<false /* pmo */, 1 /* is_reference_wrapper */> {
+ template <typename R, typename T, typename Arg>
+ inline static constexpr auto invoke(R T::*pmo, Arg &&arg)
+ TF_RETURN(lib::forward<Arg>(arg).get().*pmo)
+ };
+
+ template <>
+ struct Invoke<false /* pmo */, 2 /* otherwise */> {
+ template <typename R, typename T, typename Arg>
+ inline static constexpr auto invoke(R T::*pmo, Arg &&arg)
+ TF_RETURN((*lib::forward<Arg>(arg)).*pmo)
+ };
+
+ template <typename R, typename T, typename Arg, typename... Args>
+ inline constexpr auto invoke(R T::*f, Arg &&arg, Args &&... args)
+ TF_RETURN(
+ Invoke<std::is_function<R>::value,
+ (std::is_base_of<T, lib::decay_t<Arg>>::value
+ ? 0
+ : is_reference_wrapper<lib::decay_t<Arg>>::value
+ ? 1
+ : 2)>::invoke(f,
+ lib::forward<Arg>(arg),
+ lib::forward<Args>(args)...))
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4100)
+#endif
+ template <typename F, typename... Args>
+ inline constexpr auto invoke(F &&f, Args &&... args)
+ TF_RETURN(lib::forward<F>(f)(lib::forward<Args>(args)...))
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+ } // namespace detail
+
+ template <typename F, typename... Args>
+ inline constexpr auto invoke(F &&f, Args &&... args)
+ TF_RETURN(detail::invoke(lib::forward<F>(f),
+ lib::forward<Args>(args)...))
+
+ namespace detail {
+
+ template <typename Void, typename, typename...>
+ struct invoke_result {};
+
+ template <typename F, typename... Args>
+ struct invoke_result<void_t<decltype(lib::invoke(
+ std::declval<F>(), std::declval<Args>()...))>,
+ F,
+ Args...>
+ : identity<decltype(
+ lib::invoke(std::declval<F>(), std::declval<Args>()...))> {};
+
+ } // namespace detail
+
+ template <typename F, typename... Args>
+ using invoke_result = detail::invoke_result<void, F, Args...>;
+
+ template <typename F, typename... Args>
+ using invoke_result_t = typename invoke_result<F, Args...>::type;
+
+ namespace detail {
+
+ template <typename Void, typename, typename...>
+ struct is_invocable : std::false_type {};
+
+ template <typename F, typename... Args>
+ struct is_invocable<void_t<invoke_result_t<F, Args...>>, F, Args...>
+ : std::true_type {};
+
+ template <typename Void, typename, typename, typename...>
+ struct is_invocable_r : std::false_type {};
+
+ template <typename R, typename F, typename... Args>
+ struct is_invocable_r<void_t<invoke_result_t<F, Args...>>,
+ R,
+ F,
+ Args...>
+ : std::is_convertible<invoke_result_t<F, Args...>, R> {};
+
+ } // namespace detail
+
+ template <typename F, typename... Args>
+ using is_invocable = detail::is_invocable<void, F, Args...>;
+
+ template <typename R, typename F, typename... Args>
+ using is_invocable_r = detail::is_invocable_r<void, R, F, Args...>;
+
+ // <memory>
+#ifdef TF_BUILTIN_ADDRESSOF
+ template <typename T>
+ inline constexpr T *addressof(T &arg) noexcept {
+ return __builtin_addressof(arg);
+ }
+#else
+ namespace detail {
+
+ namespace has_addressof_impl {
+
+ struct fail;
+
+ template <typename T>
+ inline fail operator&(T &&);
+
+ template <typename T>
+ inline static constexpr bool impl() {
+ return (std::is_class<T>::value || std::is_union<T>::value) &&
+ !std::is_same<decltype(&std::declval<T &>()), fail>::value;
+ }
+
+ } // namespace has_addressof_impl
+
+ template <typename T>
+ using has_addressof = bool_constant<has_addressof_impl::impl<T>()>;
+
+ template <typename T>
+ inline constexpr T *addressof(T &arg, std::true_type) noexcept {
+ return std::addressof(arg);
+ }
+
+ template <typename T>
+ inline constexpr T *addressof(T &arg, std::false_type) noexcept {
+ return &arg;
+ }
+
+ } // namespace detail
+
+ template <typename T>
+ inline constexpr T *addressof(T &arg) noexcept {
+ return detail::addressof(arg, detail::has_addressof<T>{});
+ }
+#endif
+
+ template <typename T>
+ inline constexpr T *addressof(const T &&) = delete;
+
+ } // namespace cpp17
+
+ template <typename T>
+ struct remove_all_extents : identity<T> {};
+
+ template <typename T, std::size_t N>
+ struct remove_all_extents<array<T, N>> : remove_all_extents<T> {};
+
+ template <typename T>
+ using remove_all_extents_t = typename remove_all_extents<T>::type;
+
+ template <std::size_t N>
+ using size_constant = std::integral_constant<std::size_t, N>;
+
+ template <std::size_t I, typename T>
+ struct indexed_type : size_constant<I> { using type = T; };
+
+ template <bool... Bs>
+ using all = std::is_same<integer_sequence<bool, true, Bs...>,
+ integer_sequence<bool, Bs..., true>>;
+
+#ifdef TF_TYPE_PACK_ELEMENT
+ template <std::size_t I, typename... Ts>
+ using type_pack_element_t = __type_pack_element<I, Ts...>;
+#else
+ template <std::size_t I, typename... Ts>
+ struct type_pack_element_impl {
+ private:
+ template <typename>
+ struct set;
+
+ template <std::size_t... Is>
+ struct set<index_sequence<Is...>> : indexed_type<Is, Ts>... {};
+
+ template <typename T>
+ inline static std::enable_if<true, T> impl(indexed_type<I, T>);
+
+ inline static std::enable_if<false> impl(...);
+
+ public:
+ using type = decltype(impl(set<index_sequence_for<Ts...>>{}));
+ };
+
+ template <std::size_t I, typename... Ts>
+ using type_pack_element = typename type_pack_element_impl<I, Ts...>::type;
+
+ template <std::size_t I, typename... Ts>
+ using type_pack_element_t = typename type_pack_element<I, Ts...>::type;
+#endif
+
+#ifdef TF_TRIVIALITY_TYPE_TRAITS
+ using std::is_trivially_copy_constructible;
+ using std::is_trivially_move_constructible;
+ using std::is_trivially_copy_assignable;
+ using std::is_trivially_move_assignable;
+#else
+ template <typename T>
+ struct is_trivially_copy_constructible
+ : bool_constant<
+ std::is_copy_constructible<T>::value && __has_trivial_copy(T)> {};
+
+ template <typename T>
+ struct is_trivially_move_constructible : bool_constant<__is_trivial(T)> {};
+
+ template <typename T>
+ struct is_trivially_copy_assignable
+ : bool_constant<
+ std::is_copy_assignable<T>::value && __has_trivial_assign(T)> {};
+
+ template <typename T>
+ struct is_trivially_move_assignable : bool_constant<__is_trivial(T)> {};
+#endif
+
+ template <typename T, bool>
+ struct dependent_type : T {};
+
+ template <typename Is, std::size_t J>
+ struct push_back;
+
+ template <typename Is, std::size_t J>
+ using push_back_t = typename push_back<Is, J>::type;
+
+ template <std::size_t... Is, std::size_t J>
+ struct push_back<index_sequence<Is...>, J> {
+ using type = index_sequence<Is..., J>;
+ };
+
+ } // namespace lib
+}} // namespace tf::nstd
+
+#undef TF_RETURN
+
+
+namespace tf { namespace nstd {
+
+#ifdef TF_RETURN_TYPE_DEDUCTION
+
+#define AUTO auto
+#define AUTO_RETURN(...) { return __VA_ARGS__; }
+
+#define AUTO_REFREF auto &&
+#define AUTO_REFREF_RETURN(...) { return __VA_ARGS__; }
+
+#define DECLTYPE_AUTO decltype(auto)
+#define DECLTYPE_AUTO_RETURN(...) { return __VA_ARGS__; }
+
+#else
+
+#define AUTO auto
+#define AUTO_RETURN(...) \
+ -> lib::decay_t<decltype(__VA_ARGS__)> { return __VA_ARGS__; }
+
+#define AUTO_REFREF auto
+#define AUTO_REFREF_RETURN(...) \
+ -> decltype((__VA_ARGS__)) { \
+ static_assert(std::is_reference<decltype((__VA_ARGS__))>::value, ""); \
+ return __VA_ARGS__; \
+ }
+
+#define DECLTYPE_AUTO auto
+#define DECLTYPE_AUTO_RETURN(...) \
+ -> decltype(__VA_ARGS__) { return __VA_ARGS__; }
+
+#endif
+
+ class bad_variant_access : public std::exception {
+ public:
+ virtual const char *what() const noexcept override { return "bad_variant_access"; }
+ };
+
+ [[noreturn]] inline void throw_bad_variant_access() {
+#ifdef TF_EXCEPTIONS
+ throw bad_variant_access{};
+#else
+ std::terminate();
+ TF_BUILTIN_UNREACHABLE;
+#endif
+ }
+
+ template <typename... Ts>
+ class variant;
+
+ template <typename T>
+ struct variant_size;
+
+#ifdef TF_VARIABLE_TEMPLATES
+ template <typename T>
+ constexpr std::size_t variant_size_v = variant_size<T>::value;
+#endif
+
+ template <typename T>
+ struct variant_size<const T> : variant_size<T> {};
+
+ template <typename T>
+ struct variant_size<volatile T> : variant_size<T> {};
+
+ template <typename T>
+ struct variant_size<const volatile T> : variant_size<T> {};
+
+ template <typename... Ts>
+ struct variant_size<variant<Ts...>> : lib::size_constant<sizeof...(Ts)> {};
+
+ template <std::size_t I, typename T>
+ struct variant_alternative;
+
+ template <std::size_t I, typename T>
+ using variant_alternative_t = typename variant_alternative<I, T>::type;
+
+ template <std::size_t I, typename T>
+ struct variant_alternative<I, const T>
+ : std::add_const<variant_alternative_t<I, T>> {};
+
+ template <std::size_t I, typename T>
+ struct variant_alternative<I, volatile T>
+ : std::add_volatile<variant_alternative_t<I, T>> {};
+
+ template <std::size_t I, typename T>
+ struct variant_alternative<I, const volatile T>
+ : std::add_cv<variant_alternative_t<I, T>> {};
+
+ template <std::size_t I, typename... Ts>
+ struct variant_alternative<I, variant<Ts...>> {
+ static_assert(I < sizeof...(Ts),
+ "index out of bounds in `std::variant_alternative<>`");
+ using type = lib::type_pack_element_t<I, Ts...>;
+ };
+
+ constexpr std::size_t variant_npos = static_cast<std::size_t>(-1);
+
+ namespace detail {
+
+ constexpr std::size_t not_found = static_cast<std::size_t>(-1);
+ constexpr std::size_t ambiguous = static_cast<std::size_t>(-2);
+
+#ifdef TF_CPP14_CONSTEXPR
+ template <typename T, typename... Ts>
+ inline constexpr std::size_t find_index() {
+ constexpr lib::array<bool, sizeof...(Ts)> matches = {
+ {std::is_same<T, Ts>::value...}
+ };
+ std::size_t result = not_found;
+ for (std::size_t i = 0; i < sizeof...(Ts); ++i) {
+ if (matches[i]) {
+ if (result != not_found) {
+ return ambiguous;
+ }
+ result = i;
+ }
+ }
+ return result;
+ }
+#else
+ inline constexpr std::size_t find_index_impl(std::size_t result,
+ std::size_t) {
+ return result;
+ }
+
+ template <typename... Bs>
+ inline constexpr std::size_t find_index_impl(std::size_t result,
+ std::size_t idx,
+ bool b,
+ Bs... bs) {
+ return b ? (result != not_found ? ambiguous
+ : find_index_impl(idx, idx + 1, bs...))
+ : find_index_impl(result, idx + 1, bs...);
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr std::size_t find_index() {
+ return find_index_impl(not_found, 0, std::is_same<T, Ts>::value...);
+ }
+#endif
+
+ template <std::size_t I>
+ using find_index_sfinae_impl =
+ lib::enable_if_t<I != not_found && I != ambiguous,
+ lib::size_constant<I>>;
+
+ template <typename T, typename... Ts>
+ using find_index_sfinae = find_index_sfinae_impl<find_index<T, Ts...>()>;
+
+ template <std::size_t I>
+ struct find_index_checked_impl : lib::size_constant<I> {
+ static_assert(I != not_found, "the specified type is not found.");
+ static_assert(I != ambiguous, "the specified type is ambiguous.");
+ };
+
+ template <typename T, typename... Ts>
+ using find_index_checked = find_index_checked_impl<find_index<T, Ts...>()>;
+
+ struct valueless_t {};
+
+ enum class Trait { TriviallyAvailable, Available, Unavailable };
+
+ template <typename T,
+ template <typename> class IsTriviallyAvailable,
+ template <typename> class IsAvailable>
+ inline constexpr Trait trait() {
+ return IsTriviallyAvailable<T>::value
+ ? Trait::TriviallyAvailable
+ : IsAvailable<T>::value ? Trait::Available
+ : Trait::Unavailable;
+ }
+
+#ifdef TF_CPP14_CONSTEXPR
+ template <typename... Traits>
+ inline constexpr Trait common_trait(Traits... traits_) {
+ Trait result = Trait::TriviallyAvailable;
+ lib::array<Trait, sizeof...(Traits)> traits = {{traits_...}};
+ for (std::size_t i = 0; i < sizeof...(Traits); ++i) {
+ Trait t = traits[i];
+ if (static_cast<int>(t) > static_cast<int>(result)) {
+ result = t;
+ }
+ }
+ return result;
+ }
+#else
+ inline constexpr Trait common_trait_impl(Trait result) { return result; }
+
+ template <typename... Traits>
+ inline constexpr Trait common_trait_impl(Trait result,
+ Trait t,
+ Traits... ts) {
+ return static_cast<int>(t) > static_cast<int>(result)
+ ? common_trait_impl(t, ts...)
+ : common_trait_impl(result, ts...);
+ }
+
+ template <typename... Traits>
+ inline constexpr Trait common_trait(Traits... ts) {
+ return common_trait_impl(Trait::TriviallyAvailable, ts...);
+ }
+#endif
+
+ template <typename... Ts>
+ struct traits {
+ static constexpr Trait copy_constructible_trait =
+ common_trait(trait<Ts,
+ lib::is_trivially_copy_constructible,
+ std::is_copy_constructible>()...);
+
+ static constexpr Trait move_constructible_trait =
+ common_trait(trait<Ts,
+ lib::is_trivially_move_constructible,
+ std::is_move_constructible>()...);
+
+ static constexpr Trait copy_assignable_trait =
+ common_trait(copy_constructible_trait,
+ trait<Ts,
+ lib::is_trivially_copy_assignable,
+ std::is_copy_assignable>()...);
+
+ static constexpr Trait move_assignable_trait =
+ common_trait(move_constructible_trait,
+ trait<Ts,
+ lib::is_trivially_move_assignable,
+ std::is_move_assignable>()...);
+
+ static constexpr Trait destructible_trait =
+ common_trait(trait<Ts,
+ std::is_trivially_destructible,
+ std::is_destructible>()...);
+ };
+
+ namespace access {
+
+ struct recursive_union {
+#ifdef TF_RETURN_TYPE_DEDUCTION
+ template <typename V>
+ inline static constexpr auto &&get_alt(V &&v, in_place_index_t<0>) {
+ return lib::forward<V>(v).head_;
+ }
+
+ template <typename V, std::size_t I>
+ inline static constexpr auto &&get_alt(V &&v, in_place_index_t<I>) {
+ return get_alt(lib::forward<V>(v).tail_, in_place_index_t<I - 1>{});
+ }
+#else
+ template <std::size_t I, bool Dummy = true>
+ struct get_alt_impl {
+ template <typename V>
+ inline constexpr AUTO_REFREF operator()(V &&v) const
+ AUTO_REFREF_RETURN(get_alt_impl<I - 1>{}(lib::forward<V>(v).tail_))
+ };
+
+ template <bool Dummy>
+ struct get_alt_impl<0, Dummy> {
+ template <typename V>
+ inline constexpr AUTO_REFREF operator()(V &&v) const
+ AUTO_REFREF_RETURN(lib::forward<V>(v).head_)
+ };
+
+ template <typename V, std::size_t I>
+ inline static constexpr AUTO_REFREF get_alt(V &&v, in_place_index_t<I>)
+ AUTO_REFREF_RETURN(get_alt_impl<I>{}(lib::forward<V>(v)))
+#endif
+ };
+
+ struct base {
+ template <std::size_t I, typename V>
+ inline static constexpr AUTO_REFREF get_alt(V &&v)
+#ifdef _MSC_VER
+ AUTO_REFREF_RETURN(recursive_union::get_alt(
+ lib::forward<V>(v).data_, in_place_index_t<I>{}))
+#else
+ AUTO_REFREF_RETURN(recursive_union::get_alt(
+ data(lib::forward<V>(v)), in_place_index_t<I>{}))
+#endif
+ };
+
+ struct variant {
+ template <std::size_t I, typename V>
+ inline static constexpr AUTO_REFREF get_alt(V &&v)
+ AUTO_REFREF_RETURN(base::get_alt<I>(lib::forward<V>(v).impl_))
+ };
+
+ } // namespace access
+
+ namespace visitation {
+
+#if defined(TF_CPP14_CONSTEXPR) && !defined(_MSC_VER)
+#define TF_VARIANT_SWITCH_VISIT
+#endif
+
+ struct base {
+ template <typename Visitor, typename... Vs>
+ using dispatch_result_t = decltype(
+ lib::invoke(std::declval<Visitor>(),
+ access::base::get_alt<0>(std::declval<Vs>())...));
+
+ template <typename Expected>
+ struct expected {
+ template <typename Actual>
+ inline static constexpr bool but_got() {
+ return std::is_same<Expected, Actual>::value;
+ }
+ };
+
+ template <typename Expected, typename Actual>
+ struct visit_return_type_check {
+ static_assert(
+ expected<Expected>::template but_got<Actual>(),
+ "`visit` requires the visitor to have a single return type");
+
+ template <typename Visitor, typename... Alts>
+ inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor,
+ Alts &&... alts)
+ DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward<Visitor>(visitor),
+ lib::forward<Alts>(alts)...))
+ };
+
+#ifdef TF_VARIANT_SWITCH_VISIT
+ template <bool B, typename R, typename... ITs>
+ struct dispatcher;
+
+ template <typename R, typename... ITs>
+ struct dispatcher<false, R, ITs...> {
+ template <std::size_t B, typename F, typename... Vs>
+ TF_ALWAYS_INLINE static constexpr R dispatch(
+ F &&, typename ITs::type &&..., Vs &&...) {
+ TF_BUILTIN_UNREACHABLE;
+ }
+
+ template <std::size_t I, typename F, typename... Vs>
+ TF_ALWAYS_INLINE static constexpr R dispatch_case(F &&, Vs &&...) {
+ TF_BUILTIN_UNREACHABLE;
+ }
+
+ template <std::size_t B, typename F, typename... Vs>
+ TF_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t,
+ F &&,
+ Vs &&...) {
+ TF_BUILTIN_UNREACHABLE;
+ }
+ };
+
+ template <typename R, typename... ITs>
+ struct dispatcher<true, R, ITs...> {
+ template <std::size_t B, typename F>
+ TF_ALWAYS_INLINE static constexpr R dispatch(
+ F &&f, typename ITs::type &&... visited_vs) {
+ using Expected = R;
+ using Actual = decltype(lib::invoke(
+ lib::forward<F>(f),
+ access::base::get_alt<ITs::value>(
+ lib::forward<typename ITs::type>(visited_vs))...));
+ return visit_return_type_check<Expected, Actual>::invoke(
+ lib::forward<F>(f),
+ access::base::get_alt<ITs::value>(
+ lib::forward<typename ITs::type>(visited_vs))...);
+ }
+
+ template <std::size_t B, typename F, typename V, typename... Vs>
+ TF_ALWAYS_INLINE static constexpr R dispatch(
+ F &&f, typename ITs::type &&... visited_vs, V &&v, Vs &&... vs) {
+#define TF_DISPATCH(I) \
+ dispatcher<(I < lib::decay_t<V>::size()), \
+ R, \
+ ITs..., \
+ lib::indexed_type<I, V>>:: \
+ template dispatch<0>(lib::forward<F>(f), \
+ lib::forward<typename ITs::type>(visited_vs)..., \
+ lib::forward<V>(v), \
+ lib::forward<Vs>(vs)...)
+
+#define TF_DEFAULT(I) \
+ dispatcher<(I < lib::decay_t<V>::size()), R, ITs...>::template dispatch<I>( \
+ lib::forward<F>(f), \
+ lib::forward<typename ITs::type>(visited_vs)..., \
+ lib::forward<V>(v), \
+ lib::forward<Vs>(vs)...)
+
+ switch (v.index()) {
+ case B + 0: return TF_DISPATCH(B + 0);
+ case B + 1: return TF_DISPATCH(B + 1);
+ case B + 2: return TF_DISPATCH(B + 2);
+ case B + 3: return TF_DISPATCH(B + 3);
+ case B + 4: return TF_DISPATCH(B + 4);
+ case B + 5: return TF_DISPATCH(B + 5);
+ case B + 6: return TF_DISPATCH(B + 6);
+ case B + 7: return TF_DISPATCH(B + 7);
+ case B + 8: return TF_DISPATCH(B + 8);
+ case B + 9: return TF_DISPATCH(B + 9);
+ case B + 10: return TF_DISPATCH(B + 10);
+ case B + 11: return TF_DISPATCH(B + 11);
+ case B + 12: return TF_DISPATCH(B + 12);
+ case B + 13: return TF_DISPATCH(B + 13);
+ case B + 14: return TF_DISPATCH(B + 14);
+ case B + 15: return TF_DISPATCH(B + 15);
+ case B + 16: return TF_DISPATCH(B + 16);
+ case B + 17: return TF_DISPATCH(B + 17);
+ case B + 18: return TF_DISPATCH(B + 18);
+ case B + 19: return TF_DISPATCH(B + 19);
+ case B + 20: return TF_DISPATCH(B + 20);
+ case B + 21: return TF_DISPATCH(B + 21);
+ case B + 22: return TF_DISPATCH(B + 22);
+ case B + 23: return TF_DISPATCH(B + 23);
+ case B + 24: return TF_DISPATCH(B + 24);
+ case B + 25: return TF_DISPATCH(B + 25);
+ case B + 26: return TF_DISPATCH(B + 26);
+ case B + 27: return TF_DISPATCH(B + 27);
+ case B + 28: return TF_DISPATCH(B + 28);
+ case B + 29: return TF_DISPATCH(B + 29);
+ case B + 30: return TF_DISPATCH(B + 30);
+ case B + 31: return TF_DISPATCH(B + 31);
+ default: return TF_DEFAULT(B + 32);
+ }
+
+#undef TF_DEFAULT
+#undef TF_DISPATCH
+ }
+
+ template <std::size_t I, typename F, typename... Vs>
+ TF_ALWAYS_INLINE static constexpr R dispatch_case(F &&f,
+ Vs &&... vs) {
+ using Expected = R;
+ using Actual = decltype(
+ lib::invoke(lib::forward<F>(f),
+ access::base::get_alt<I>(lib::forward<Vs>(vs))...));
+ return visit_return_type_check<Expected, Actual>::invoke(
+ lib::forward<F>(f),
+ access::base::get_alt<I>(lib::forward<Vs>(vs))...);
+ }
+
+ template <std::size_t B, typename F, typename V, typename... Vs>
+ TF_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t index,
+ F &&f,
+ V &&v,
+ Vs &&... vs) {
+ static_assert(lib::all<(lib::decay_t<V>::size() ==
+ lib::decay_t<Vs>::size())...>::value,
+ "all of the variants must be the same size.");
+#define TF_DISPATCH_AT(I) \
+ dispatcher<(I < lib::decay_t<V>::size()), R>::template dispatch_case<I>( \
+ lib::forward<F>(f), lib::forward<V>(v), lib::forward<Vs>(vs)...)
+
+#define TF_DEFAULT(I) \
+ dispatcher<(I < lib::decay_t<V>::size()), R>::template dispatch_at<I>( \
+ index, lib::forward<F>(f), lib::forward<V>(v), lib::forward<Vs>(vs)...)
+
+ switch (index) {
+ case B + 0: return TF_DISPATCH_AT(B + 0);
+ case B + 1: return TF_DISPATCH_AT(B + 1);
+ case B + 2: return TF_DISPATCH_AT(B + 2);
+ case B + 3: return TF_DISPATCH_AT(B + 3);
+ case B + 4: return TF_DISPATCH_AT(B + 4);
+ case B + 5: return TF_DISPATCH_AT(B + 5);
+ case B + 6: return TF_DISPATCH_AT(B + 6);
+ case B + 7: return TF_DISPATCH_AT(B + 7);
+ case B + 8: return TF_DISPATCH_AT(B + 8);
+ case B + 9: return TF_DISPATCH_AT(B + 9);
+ case B + 10: return TF_DISPATCH_AT(B + 10);
+ case B + 11: return TF_DISPATCH_AT(B + 11);
+ case B + 12: return TF_DISPATCH_AT(B + 12);
+ case B + 13: return TF_DISPATCH_AT(B + 13);
+ case B + 14: return TF_DISPATCH_AT(B + 14);
+ case B + 15: return TF_DISPATCH_AT(B + 15);
+ case B + 16: return TF_DISPATCH_AT(B + 16);
+ case B + 17: return TF_DISPATCH_AT(B + 17);
+ case B + 18: return TF_DISPATCH_AT(B + 18);
+ case B + 19: return TF_DISPATCH_AT(B + 19);
+ case B + 20: return TF_DISPATCH_AT(B + 20);
+ case B + 21: return TF_DISPATCH_AT(B + 21);
+ case B + 22: return TF_DISPATCH_AT(B + 22);
+ case B + 23: return TF_DISPATCH_AT(B + 23);
+ case B + 24: return TF_DISPATCH_AT(B + 24);
+ case B + 25: return TF_DISPATCH_AT(B + 25);
+ case B + 26: return TF_DISPATCH_AT(B + 26);
+ case B + 27: return TF_DISPATCH_AT(B + 27);
+ case B + 28: return TF_DISPATCH_AT(B + 28);
+ case B + 29: return TF_DISPATCH_AT(B + 29);
+ case B + 30: return TF_DISPATCH_AT(B + 30);
+ case B + 31: return TF_DISPATCH_AT(B + 31);
+ default: return TF_DEFAULT(B + 32);
+ }
+
+#undef TF_DEFAULT
+#undef TF_DISPATCH_AT
+ }
+ };
+#else
+ template <typename T>
+ inline static constexpr const T &at(const T &elem) noexcept {
+ return elem;
+ }
+
+ template <typename T, std::size_t N, typename... Is>
+ inline static constexpr const lib::remove_all_extents_t<T> &at(
+ const lib::array<T, N> &elems, std::size_t i, Is... is) noexcept {
+ return at(elems[i], is...);
+ }
+
+ template <typename F, typename... Fs>
+ inline static constexpr lib::array<lib::decay_t<F>, sizeof...(Fs) + 1>
+ make_farray(F &&f, Fs &&... fs) {
+ return {{lib::forward<F>(f), lib::forward<Fs>(fs)...}};
+ }
+
+ template <typename F, typename... Vs>
+ struct make_fmatrix_impl {
+
+ template <std::size_t... Is>
+ inline static constexpr dispatch_result_t<F, Vs...> dispatch(
+ F &&f, Vs &&... vs) {
+ using Expected = dispatch_result_t<F, Vs...>;
+ using Actual = decltype(lib::invoke(
+ lib::forward<F>(f),
+ access::base::get_alt<Is>(lib::forward<Vs>(vs))...));
+ return visit_return_type_check<Expected, Actual>::invoke(
+ lib::forward<F>(f),
+ access::base::get_alt<Is>(lib::forward<Vs>(vs))...);
+ }
+
+#ifdef TF_RETURN_TYPE_DEDUCTION
+ template <std::size_t... Is>
+ inline static constexpr auto impl(lib::index_sequence<Is...>) {
+ return &dispatch<Is...>;
+ }
+
+ template <typename Is, std::size_t... Js, typename... Ls>
+ inline static constexpr auto impl(Is,
+ lib::index_sequence<Js...>,
+ Ls... ls) {
+ return make_farray(impl(lib::push_back_t<Is, Js>{}, ls...)...);
+ }
+#else
+ template <typename...>
+ struct impl;
+
+ template <std::size_t... Is>
+ struct impl<lib::index_sequence<Is...>> {
+ inline constexpr AUTO operator()() const
+ AUTO_RETURN(&dispatch<Is...>)
+ };
+
+ template <typename Is, std::size_t... Js, typename... Ls>
+ struct impl<Is, lib::index_sequence<Js...>, Ls...> {
+ inline constexpr AUTO operator()() const
+ AUTO_RETURN(
+ make_farray(impl<lib::push_back_t<Is, Js>, Ls...>{}()...))
+ };
+#endif
+ };
+
+#ifdef TF_RETURN_TYPE_DEDUCTION
+ template <typename F, typename... Vs>
+ inline static constexpr auto make_fmatrix() {
+ return make_fmatrix_impl<F, Vs...>::impl(
+ lib::index_sequence<>{},
+ lib::make_index_sequence<lib::decay_t<Vs>::size()>{}...);
+ }
+#else
+ template <typename F, typename... Vs>
+ inline static constexpr AUTO make_fmatrix()
+ AUTO_RETURN(
+ typename make_fmatrix_impl<F, Vs...>::template impl<
+ lib::index_sequence<>,
+ lib::make_index_sequence<lib::decay_t<Vs>::size()>...>{}())
+#endif
+
+ template <typename F, typename... Vs>
+ struct make_fdiagonal_impl {
+ template <std::size_t I>
+ inline static constexpr dispatch_result_t<F, Vs...> dispatch(
+ F &&f, Vs &&... vs) {
+ using Expected = dispatch_result_t<F, Vs...>;
+ using Actual = decltype(
+ lib::invoke(lib::forward<F>(f),
+ access::base::get_alt<I>(lib::forward<Vs>(vs))...));
+ return visit_return_type_check<Expected, Actual>::invoke(
+ lib::forward<F>(f),
+ access::base::get_alt<I>(lib::forward<Vs>(vs))...);
+ }
+
+ template <std::size_t... Is>
+ inline static constexpr AUTO impl(lib::index_sequence<Is...>)
+ AUTO_RETURN(make_farray(&dispatch<Is>...))
+ };
+
+ template <typename F, typename V, typename... Vs>
+ inline static constexpr auto make_fdiagonal()
+ -> decltype(make_fdiagonal_impl<F, V, Vs...>::impl(
+ lib::make_index_sequence<lib::decay_t<V>::size()>{})) {
+ static_assert(lib::all<(lib::decay_t<V>::size() ==
+ lib::decay_t<Vs>::size())...>::value,
+ "all of the variants must be the same size.");
+ return make_fdiagonal_impl<F, V, Vs...>::impl(
+ lib::make_index_sequence<lib::decay_t<V>::size()>{});
+ }
+#endif
+ };
+
+#if !defined(TF_VARIANT_SWITCH_VISIT) && \
+ (!defined(_MSC_VER) || _MSC_VER >= 1910)
+ template <typename F, typename... Vs>
+ using fmatrix_t = decltype(base::make_fmatrix<F, Vs...>());
+
+ template <typename F, typename... Vs>
+ struct fmatrix {
+ static constexpr fmatrix_t<F, Vs...> value =
+ base::make_fmatrix<F, Vs...>();
+ };
+
+ template <typename F, typename... Vs>
+ constexpr fmatrix_t<F, Vs...> fmatrix<F, Vs...>::value;
+
+ template <typename F, typename... Vs>
+ using fdiagonal_t = decltype(base::make_fdiagonal<F, Vs...>());
+
+ template <typename F, typename... Vs>
+ struct fdiagonal {
+ static constexpr fdiagonal_t<F, Vs...> value =
+ base::make_fdiagonal<F, Vs...>();
+ };
+
+ template <typename F, typename... Vs>
+ constexpr fdiagonal_t<F, Vs...> fdiagonal<F, Vs...>::value;
+#endif
+
+ struct alt {
+ template <typename Visitor, typename... Vs>
+ inline static constexpr DECLTYPE_AUTO visit_alt(Visitor &&visitor,
+ Vs &&... vs)
+#ifdef TF_VARIANT_SWITCH_VISIT
+ DECLTYPE_AUTO_RETURN(
+ base::dispatcher<
+ true,
+ base::dispatch_result_t<Visitor,
+ decltype(as_base(
+ lib::forward<Vs>(vs)))...>>::
+ template dispatch<0>(lib::forward<Visitor>(visitor),
+ as_base(lib::forward<Vs>(vs))...))
+#elif !defined(_MSC_VER) || _MSC_VER >= 1910
+ DECLTYPE_AUTO_RETURN(base::at(
+ fmatrix<Visitor &&,
+ decltype(as_base(lib::forward<Vs>(vs)))...>::value,
+ vs.index()...)(lib::forward<Visitor>(visitor),
+ as_base(lib::forward<Vs>(vs))...))
+#else
+ DECLTYPE_AUTO_RETURN(base::at(
+ base::make_fmatrix<Visitor &&,
+ decltype(as_base(lib::forward<Vs>(vs)))...>(),
+ vs.index()...)(lib::forward<Visitor>(visitor),
+ as_base(lib::forward<Vs>(vs))...))
+#endif
+
+ template <typename Visitor, typename... Vs>
+ inline static constexpr DECLTYPE_AUTO visit_alt_at(std::size_t index,
+ Visitor &&visitor,
+ Vs &&... vs)
+#ifdef TF_VARIANT_SWITCH_VISIT
+ DECLTYPE_AUTO_RETURN(
+ base::dispatcher<
+ true,
+ base::dispatch_result_t<Visitor,
+ decltype(as_base(
+ lib::forward<Vs>(vs)))...>>::
+ template dispatch_at<0>(index,
+ lib::forward<Visitor>(visitor),
+ as_base(lib::forward<Vs>(vs))...))
+#elif !defined(_MSC_VER) || _MSC_VER >= 1910
+ DECLTYPE_AUTO_RETURN(base::at(
+ fdiagonal<Visitor &&,
+ decltype(as_base(lib::forward<Vs>(vs)))...>::value,
+ index)(lib::forward<Visitor>(visitor),
+ as_base(lib::forward<Vs>(vs))...))
+#else
+ DECLTYPE_AUTO_RETURN(base::at(
+ base::make_fdiagonal<Visitor &&,
+ decltype(as_base(lib::forward<Vs>(vs)))...>(),
+ index)(lib::forward<Visitor>(visitor),
+ as_base(lib::forward<Vs>(vs))...))
+#endif
+ };
+
+ struct variant {
+ private:
+ template <typename Visitor>
+ struct visitor {
+ template <typename... Values>
+ inline static constexpr bool does_not_handle() {
+ return lib::is_invocable<Visitor, Values...>::value;
+ }
+ };
+
+ template <typename Visitor, typename... Values>
+ struct visit_exhaustiveness_check {
+ static_assert(visitor<Visitor>::template does_not_handle<Values...>(),
+ "`visit` requires the visitor to be exhaustive.");
+
+ inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor,
+ Values &&... values)
+ DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward<Visitor>(visitor),
+ lib::forward<Values>(values)...))
+ };
+
+ template <typename Visitor>
+ struct value_visitor {
+ Visitor &&visitor_;
+
+ template <typename... Alts>
+ inline constexpr DECLTYPE_AUTO operator()(Alts &&... alts) const
+ DECLTYPE_AUTO_RETURN(
+ visit_exhaustiveness_check<
+ Visitor,
+ decltype((lib::forward<Alts>(alts).value))...>::
+ invoke(lib::forward<Visitor>(visitor_),
+ lib::forward<Alts>(alts).value...))
+ };
+
+ template <typename Visitor>
+ inline static constexpr AUTO make_value_visitor(Visitor &&visitor)
+ AUTO_RETURN(value_visitor<Visitor>{lib::forward<Visitor>(visitor)})
+
+ public:
+ template <typename Visitor, typename... Vs>
+ inline static constexpr DECLTYPE_AUTO visit_alt(Visitor &&visitor,
+ Vs &&... vs)
+ DECLTYPE_AUTO_RETURN(alt::visit_alt(lib::forward<Visitor>(visitor),
+ lib::forward<Vs>(vs).impl_...))
+
+ template <typename Visitor, typename... Vs>
+ inline static constexpr DECLTYPE_AUTO visit_alt_at(std::size_t index,
+ Visitor &&visitor,
+ Vs &&... vs)
+ DECLTYPE_AUTO_RETURN(
+ alt::visit_alt_at(index,
+ lib::forward<Visitor>(visitor),
+ lib::forward<Vs>(vs).impl_...))
+
+ template <typename Visitor, typename... Vs>
+ inline static constexpr DECLTYPE_AUTO visit_value(Visitor &&visitor,
+ Vs &&... vs)
+ DECLTYPE_AUTO_RETURN(
+ visit_alt(make_value_visitor(lib::forward<Visitor>(visitor)),
+ lib::forward<Vs>(vs)...))
+
+ template <typename Visitor, typename... Vs>
+ inline static constexpr DECLTYPE_AUTO visit_value_at(std::size_t index,
+ Visitor &&visitor,
+ Vs &&... vs)
+ DECLTYPE_AUTO_RETURN(
+ visit_alt_at(index,
+ make_value_visitor(lib::forward<Visitor>(visitor)),
+ lib::forward<Vs>(vs)...))
+ };
+
+ } // namespace visitation
+
+ template <std::size_t Index, typename T>
+ struct alt {
+ using value_type = T;
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4244)
+#endif
+ template <typename... Args>
+ inline explicit constexpr alt(in_place_t, Args &&... args)
+ : value(lib::forward<Args>(args)...) {}
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+ T value;
+ };
+
+ template <Trait DestructibleTrait, std::size_t Index, typename... Ts>
+ union recursive_union;
+
+ template <Trait DestructibleTrait, std::size_t Index>
+ union recursive_union<DestructibleTrait, Index> {};
+
+#define TF_VARIANT_RECURSIVE_UNION(destructible_trait, destructor) \
+ template <std::size_t Index, typename T, typename... Ts> \
+ union recursive_union<destructible_trait, Index, T, Ts...> { \
+ public: \
+ inline explicit constexpr recursive_union(valueless_t) noexcept \
+ : dummy_{} {} \
+ \
+ template <typename... Args> \
+ inline explicit constexpr recursive_union(in_place_index_t<0>, \
+ Args &&... args) \
+ : head_(in_place_t{}, lib::forward<Args>(args)...) {} \
+ \
+ template <std::size_t I, typename... Args> \
+ inline explicit constexpr recursive_union(in_place_index_t<I>, \
+ Args &&... args) \
+ : tail_(in_place_index_t<I - 1>{}, lib::forward<Args>(args)...) {} \
+ \
+ recursive_union(const recursive_union &) = default; \
+ recursive_union(recursive_union &&) = default; \
+ \
+ destructor \
+ \
+ recursive_union &operator=(const recursive_union &) = default; \
+ recursive_union &operator=(recursive_union &&) = default; \
+ \
+ private: \
+ char dummy_; \
+ alt<Index, T> head_; \
+ recursive_union<destructible_trait, Index + 1, Ts...> tail_; \
+ \
+ friend struct access::recursive_union; \
+ }
+
+ TF_VARIANT_RECURSIVE_UNION(Trait::TriviallyAvailable,
+ ~recursive_union() = default;);
+ TF_VARIANT_RECURSIVE_UNION(Trait::Available,
+ ~recursive_union() {});
+ TF_VARIANT_RECURSIVE_UNION(Trait::Unavailable,
+ ~recursive_union() = delete;);
+
+#undef TF_VARIANT_RECURSIVE_UNION
+
+ template <typename... Ts>
+ using index_t = typename std::conditional<
+ sizeof...(Ts) < (std::numeric_limits<unsigned char>::max)(),
+ unsigned char,
+ typename std::conditional<
+ sizeof...(Ts) < (std::numeric_limits<unsigned short>::max)(),
+ unsigned short,
+ unsigned int>::type
+ >::type;
+
+ template <Trait DestructibleTrait, typename... Ts>
+ class base {
+ public:
+ inline explicit constexpr base(valueless_t tag) noexcept
+ : data_(tag), index_(static_cast<index_t<Ts...>>(-1)) {}
+
+ template <std::size_t I, typename... Args>
+ inline explicit constexpr base(in_place_index_t<I>, Args &&... args)
+ : data_(in_place_index_t<I>{}, lib::forward<Args>(args)...),
+ index_(I) {}
+
+ inline constexpr bool valueless_by_exception() const noexcept {
+ return index_ == static_cast<index_t<Ts...>>(-1);
+ }
+
+ inline constexpr std::size_t index() const noexcept {
+ return valueless_by_exception() ? variant_npos : index_;
+ }
+
+ protected:
+ using data_t = recursive_union<DestructibleTrait, 0, Ts...>;
+
+ friend inline constexpr base &as_base(base &b) { return b; }
+ friend inline constexpr const base &as_base(const base &b) { return b; }
+ friend inline constexpr base &&as_base(base &&b) { return lib::move(b); }
+ friend inline constexpr const base &&as_base(const base &&b) { return lib::move(b); }
+
+ friend inline constexpr data_t &data(base &b) { return b.data_; }
+ friend inline constexpr const data_t &data(const base &b) { return b.data_; }
+ friend inline constexpr data_t &&data(base &&b) { return lib::move(b).data_; }
+ friend inline constexpr const data_t &&data(const base &&b) { return lib::move(b).data_; }
+
+ inline static constexpr std::size_t size() { return sizeof...(Ts); }
+
+ data_t data_;
+ index_t<Ts...> index_;
+
+ friend struct access::base;
+ friend struct visitation::base;
+ };
+
+ struct dtor {
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4100)
+#endif
+ template <typename Alt>
+ inline void operator()(Alt &alt) const noexcept { alt.~Alt(); }
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+ };
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1910
+#define TF_INHERITING_CTOR(type, base) using base::base;
+#else
+#define TF_INHERITING_CTOR(type, base) \
+ template <typename... Args> \
+ inline explicit constexpr type(Args &&... args) \
+ : base(lib::forward<Args>(args)...) {}
+#endif
+
+ template <typename Traits, Trait = Traits::destructible_trait>
+ class destructor;
+
+#define TF_VARIANT_DESTRUCTOR(destructible_trait, definition, destroy) \
+ template <typename... Ts> \
+ class destructor<traits<Ts...>, destructible_trait> \
+ : public base<destructible_trait, Ts...> { \
+ using super = base<destructible_trait, Ts...>; \
+ \
+ public: \
+ TF_INHERITING_CTOR(destructor, super) \
+ using super::operator=; \
+ \
+ destructor(const destructor &) = default; \
+ destructor(destructor &&) = default; \
+ definition \
+ destructor &operator=(const destructor &) = default; \
+ destructor &operator=(destructor &&) = default; \
+ \
+ protected: \
+ destroy \
+ }
+
+ TF_VARIANT_DESTRUCTOR(
+ Trait::TriviallyAvailable,
+ ~destructor() = default;,
+ inline void destroy() noexcept {
+ this->index_ = static_cast<index_t<Ts...>>(-1);
+ });
+
+ TF_VARIANT_DESTRUCTOR(
+ Trait::Available,
+ ~destructor() { destroy(); },
+ inline void destroy() noexcept {
+ if (!this->valueless_by_exception()) {
+ visitation::alt::visit_alt(dtor{}, *this);
+ }
+ this->index_ = static_cast<index_t<Ts...>>(-1);
+ });
+
+ TF_VARIANT_DESTRUCTOR(
+ Trait::Unavailable,
+ ~destructor() = delete;,
+ inline void destroy() noexcept = delete;);
+
+#undef TF_VARIANT_DESTRUCTOR
+
+ template <typename Traits>
+ class constructor : public destructor<Traits> {
+ using super = destructor<Traits>;
+
+ public:
+ TF_INHERITING_CTOR(constructor, super)
+ using super::operator=;
+
+ protected:
+#ifndef TF_GENERIC_LAMBDAS
+ struct ctor {
+ template <typename LhsAlt, typename RhsAlt>
+ inline void operator()(LhsAlt &lhs_alt, RhsAlt &&rhs_alt) const {
+ constructor::construct_alt(lhs_alt,
+ lib::forward<RhsAlt>(rhs_alt).value);
+ }
+ };
+#endif
+
+ template <std::size_t I, typename T, typename... Args>
+ inline static T &construct_alt(alt<I, T> &a, Args &&... args) {
+ auto *result = ::new (static_cast<void *>(lib::addressof(a)))
+ alt<I, T>(in_place_t{}, lib::forward<Args>(args)...);
+ return result->value;
+ }
+
+ template <typename Rhs>
+ inline static void generic_construct(constructor &lhs, Rhs &&rhs) {
+ lhs.destroy();
+ if (!rhs.valueless_by_exception()) {
+ visitation::alt::visit_alt_at(
+ rhs.index(),
+#ifdef TF_GENERIC_LAMBDAS
+ [](auto &lhs_alt, auto &&rhs_alt) {
+ constructor::construct_alt(
+ lhs_alt, lib::forward<decltype(rhs_alt)>(rhs_alt).value);
+ }
+#else
+ ctor{}
+#endif
+ ,
+ lhs,
+ lib::forward<Rhs>(rhs));
+ lhs.index_ = rhs.index_;
+ }
+ }
+ };
+
+ template <typename Traits, Trait = Traits::move_constructible_trait>
+ class move_constructor;
+
+#define TF_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, definition) \
+ template <typename... Ts> \
+ class move_constructor<traits<Ts...>, move_constructible_trait> \
+ : public constructor<traits<Ts...>> { \
+ using super = constructor<traits<Ts...>>; \
+ \
+ public: \
+ TF_INHERITING_CTOR(move_constructor, super) \
+ using super::operator=; \
+ \
+ move_constructor(const move_constructor &) = default; \
+ definition \
+ ~move_constructor() = default; \
+ move_constructor &operator=(const move_constructor &) = default; \
+ move_constructor &operator=(move_constructor &&) = default; \
+ }
+
+ TF_VARIANT_MOVE_CONSTRUCTOR(
+ Trait::TriviallyAvailable,
+ move_constructor(move_constructor &&that) = default;);
+
+ TF_VARIANT_MOVE_CONSTRUCTOR(
+ Trait::Available,
+ move_constructor(move_constructor &&that) noexcept(
+ lib::all<std::is_nothrow_move_constructible<Ts>::value...>::value)
+ : move_constructor(valueless_t{}) {
+ this->generic_construct(*this, lib::move(that));
+ });
+
+ TF_VARIANT_MOVE_CONSTRUCTOR(
+ Trait::Unavailable,
+ move_constructor(move_constructor &&) = delete;);
+
+#undef TF_VARIANT_MOVE_CONSTRUCTOR
+
+ template <typename Traits, Trait = Traits::copy_constructible_trait>
+ class copy_constructor;
+
+#define TF_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, definition) \
+ template <typename... Ts> \
+ class copy_constructor<traits<Ts...>, copy_constructible_trait> \
+ : public move_constructor<traits<Ts...>> { \
+ using super = move_constructor<traits<Ts...>>; \
+ \
+ public: \
+ TF_INHERITING_CTOR(copy_constructor, super) \
+ using super::operator=; \
+ \
+ definition \
+ copy_constructor(copy_constructor &&) = default; \
+ ~copy_constructor() = default; \
+ copy_constructor &operator=(const copy_constructor &) = default; \
+ copy_constructor &operator=(copy_constructor &&) = default; \
+ }
+
+ TF_VARIANT_COPY_CONSTRUCTOR(
+ Trait::TriviallyAvailable,
+ copy_constructor(const copy_constructor &that) = default;);
+
+ TF_VARIANT_COPY_CONSTRUCTOR(
+ Trait::Available,
+ copy_constructor(const copy_constructor &that)
+ : copy_constructor(valueless_t{}) {
+ this->generic_construct(*this, that);
+ });
+
+ TF_VARIANT_COPY_CONSTRUCTOR(
+ Trait::Unavailable,
+ copy_constructor(const copy_constructor &) = delete;);
+
+#undef TF_VARIANT_COPY_CONSTRUCTOR
+
+ template <typename Traits>
+ class assignment : public copy_constructor<Traits> {
+ using super = copy_constructor<Traits>;
+
+ public:
+ TF_INHERITING_CTOR(assignment, super)
+ using super::operator=;
+
+ template <std::size_t I, typename... Args>
+ inline /* auto & */ auto emplace(Args &&... args)
+ -> decltype(this->construct_alt(access::base::get_alt<I>(*this),
+ lib::forward<Args>(args)...)) {
+ this->destroy();
+ auto &result = this->construct_alt(access::base::get_alt<I>(*this),
+ lib::forward<Args>(args)...);
+ this->index_ = I;
+ return result;
+ }
+
+ protected:
+#ifndef TF_GENERIC_LAMBDAS
+ template <typename That>
+ struct assigner {
+ template <typename ThisAlt, typename ThatAlt>
+ inline void operator()(ThisAlt &this_alt, ThatAlt &&that_alt) const {
+ self->assign_alt(this_alt, lib::forward<ThatAlt>(that_alt).value);
+ }
+ assignment *self;
+ };
+#endif
+
+ template <std::size_t I, typename T, typename Arg>
+ inline void assign_alt(alt<I, T> &a, Arg &&arg) {
+ if (this->index() == I) {
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4244)
+#endif
+ a.value = lib::forward<Arg>(arg);
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+ } else {
+ struct {
+ void operator()(std::true_type) const {
+ this_->emplace<I>(lib::forward<Arg>(arg_));
+ }
+ void operator()(std::false_type) const {
+ this_->emplace<I>(T(lib::forward<Arg>(arg_)));
+ }
+ assignment *this_;
+ Arg &&arg_;
+ } impl{this, lib::forward<Arg>(arg)};
+ impl(lib::bool_constant<
+ std::is_nothrow_constructible<T, Arg>::value ||
+ !std::is_nothrow_move_constructible<T>::value>{});
+ }
+ }
+
+ template <typename That>
+ inline void generic_assign(That &&that) {
+ if (this->valueless_by_exception() && that.valueless_by_exception()) {
+ // do nothing.
+ } else if (that.valueless_by_exception()) {
+ this->destroy();
+ } else {
+ visitation::alt::visit_alt_at(
+ that.index(),
+#ifdef TF_GENERIC_LAMBDAS
+ [this](auto &this_alt, auto &&that_alt) {
+ this->assign_alt(
+ this_alt, lib::forward<decltype(that_alt)>(that_alt).value);
+ }
+#else
+ assigner<That>{this}
+#endif
+ ,
+ *this,
+ lib::forward<That>(that));
+ }
+ }
+ };
+
+ template <typename Traits, Trait = Traits::move_assignable_trait>
+ class move_assignment;
+
+#define TF_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, definition) \
+ template <typename... Ts> \
+ class move_assignment<traits<Ts...>, move_assignable_trait> \
+ : public assignment<traits<Ts...>> { \
+ using super = assignment<traits<Ts...>>; \
+ \
+ public: \
+ TF_INHERITING_CTOR(move_assignment, super) \
+ using super::operator=; \
+ \
+ move_assignment(const move_assignment &) = default; \
+ move_assignment(move_assignment &&) = default; \
+ ~move_assignment() = default; \
+ move_assignment &operator=(const move_assignment &) = default; \
+ definition \
+ }
+
+ TF_VARIANT_MOVE_ASSIGNMENT(
+ Trait::TriviallyAvailable,
+ move_assignment &operator=(move_assignment &&that) = default;);
+
+ TF_VARIANT_MOVE_ASSIGNMENT(
+ Trait::Available,
+ move_assignment &
+ operator=(move_assignment &&that) noexcept(
+ lib::all<(std::is_nothrow_move_constructible<Ts>::value &&
+ std::is_nothrow_move_assignable<Ts>::value)...>::value) {
+ this->generic_assign(lib::move(that));
+ return *this;
+ });
+
+ TF_VARIANT_MOVE_ASSIGNMENT(
+ Trait::Unavailable,
+ move_assignment &operator=(move_assignment &&) = delete;);
+
+#undef TF_VARIANT_MOVE_ASSIGNMENT
+
+ template <typename Traits, Trait = Traits::copy_assignable_trait>
+ class copy_assignment;
+
+#define TF_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, definition) \
+ template <typename... Ts> \
+ class copy_assignment<traits<Ts...>, copy_assignable_trait> \
+ : public move_assignment<traits<Ts...>> { \
+ using super = move_assignment<traits<Ts...>>; \
+ \
+ public: \
+ TF_INHERITING_CTOR(copy_assignment, super) \
+ using super::operator=; \
+ \
+ copy_assignment(const copy_assignment &) = default; \
+ copy_assignment(copy_assignment &&) = default; \
+ ~copy_assignment() = default; \
+ definition \
+ copy_assignment &operator=(copy_assignment &&) = default; \
+ }
+
+ TF_VARIANT_COPY_ASSIGNMENT(
+ Trait::TriviallyAvailable,
+ copy_assignment &operator=(const copy_assignment &that) = default;);
+
+ TF_VARIANT_COPY_ASSIGNMENT(
+ Trait::Available,
+ copy_assignment &operator=(const copy_assignment &that) {
+ this->generic_assign(that);
+ return *this;
+ });
+
+ TF_VARIANT_COPY_ASSIGNMENT(
+ Trait::Unavailable,
+ copy_assignment &operator=(const copy_assignment &) = delete;);
+
+#undef TF_VARIANT_COPY_ASSIGNMENT
+
+ template <typename... Ts>
+ class impl : public copy_assignment<traits<Ts...>> {
+ using super = copy_assignment<traits<Ts...>>;
+
+ public:
+ TF_INHERITING_CTOR(impl, super)
+ using super::operator=;
+
+ impl(const impl&) = default;
+ impl(impl&&) = default;
+ ~impl() = default;
+ impl &operator=(const impl &) = default;
+ impl &operator=(impl &&) = default;
+
+ template <std::size_t I, typename Arg>
+ inline void assign(Arg &&arg) {
+ this->assign_alt(access::base::get_alt<I>(*this),
+ lib::forward<Arg>(arg));
+ }
+
+ inline void swap(impl &that) {
+ if (this->valueless_by_exception() && that.valueless_by_exception()) {
+ // do nothing.
+ } else if (this->index() == that.index()) {
+ visitation::alt::visit_alt_at(this->index(),
+#ifdef TF_GENERIC_LAMBDAS
+ [](auto &this_alt, auto &that_alt) {
+ using std::swap;
+ swap(this_alt.value,
+ that_alt.value);
+ }
+#else
+ swapper{}
+#endif
+ ,
+ *this,
+ that);
+ } else {
+ impl *lhs = this;
+ impl *rhs = lib::addressof(that);
+ if (lhs->move_nothrow() && !rhs->move_nothrow()) {
+ std::swap(lhs, rhs);
+ }
+ impl tmp(lib::move(*rhs));
+#ifdef TF_EXCEPTIONS
+ // EXTENSION: When the move construction of `lhs` into `rhs` throws
+ // and `tmp` is nothrow move constructible then we move `tmp` back
+ // into `rhs` and provide the strong exception safety guarantee.
+ try {
+ this->generic_construct(*rhs, lib::move(*lhs));
+ } catch (...) {
+ if (tmp.move_nothrow()) {
+ this->generic_construct(*rhs, lib::move(tmp));
+ }
+ throw;
+ }
+#else
+ this->generic_construct(*rhs, lib::move(*lhs));
+#endif
+ this->generic_construct(*lhs, lib::move(tmp));
+ }
+ }
+
+ private:
+#ifndef TF_GENERIC_LAMBDAS
+ struct swapper {
+ template <typename ThisAlt, typename ThatAlt>
+ inline void operator()(ThisAlt &this_alt, ThatAlt &that_alt) const {
+ using std::swap;
+ swap(this_alt.value, that_alt.value);
+ }
+ };
+#endif
+
+ inline constexpr bool move_nothrow() const {
+ return this->valueless_by_exception() ||
+ lib::array<bool, sizeof...(Ts)>{
+ {std::is_nothrow_move_constructible<Ts>::value...}
+ }[this->index()];
+ }
+ };
+
+#undef TF_INHERITING_CTOR
+
+ template <typename From, typename To>
+ struct is_non_narrowing_convertible {
+ template <typename T>
+ static std::true_type test(T(&&)[1]);
+
+ template <typename T>
+ static auto impl(int) -> decltype(test<T>({std::declval<From>()}));
+
+ template <typename>
+ static auto impl(...) -> std::false_type;
+
+ static constexpr bool value = decltype(impl<To>(0))::value;
+ };
+
+ template <typename Arg,
+ std::size_t I,
+ typename T,
+ bool = std::is_arithmetic<T>::value,
+ typename = void>
+ struct overload_leaf {};
+
+ template <typename Arg, std::size_t I, typename T>
+ struct overload_leaf<Arg, I, T, false> {
+ using impl = lib::size_constant<I> (*)(T);
+ operator impl() const { return nullptr; };
+ };
+
+ template <typename Arg, std::size_t I, typename T>
+ struct overload_leaf<
+ Arg,
+ I,
+ T,
+ true
+#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ >= 5
+ ,
+ lib::enable_if_t<
+ std::is_same<lib::remove_cvref_t<T>, bool>::value
+ ? std::is_same<lib::remove_cvref_t<Arg>, bool>::value
+ : is_non_narrowing_convertible<Arg, T>::value>
+#endif
+ > {
+ using impl = lib::size_constant<I> (*)(T);
+ operator impl() const { return nullptr; };
+ };
+
+ template <typename Arg, typename... Ts>
+ struct overload_impl {
+ private:
+ template <typename>
+ struct impl;
+
+ template <std::size_t... Is>
+ struct impl<lib::index_sequence<Is...>> : overload_leaf<Arg, Is, Ts>... {};
+
+ public:
+ using type = impl<lib::index_sequence_for<Ts...>>;
+ };
+
+ template <typename Arg, typename... Ts>
+ using overload = typename overload_impl<Arg, Ts...>::type;
+
+ template <typename Arg, typename... Ts>
+ using best_match = lib::invoke_result_t<overload<Arg, Ts...>, Arg>;
+
+ template <typename T>
+ struct is_in_place_index : std::false_type {};
+
+ template <std::size_t I>
+ struct is_in_place_index<in_place_index_t<I>> : std::true_type {};
+
+ template <typename T>
+ struct is_in_place_type : std::false_type {};
+
+ template <typename T>
+ struct is_in_place_type<in_place_type_t<T>> : std::true_type {};
+
+ } // detail
+
+ template <typename... Ts>
+ class variant {
+ static_assert(0 < sizeof...(Ts),
+ "variant must consist of at least one alternative.");
+
+ static_assert(lib::all<!std::is_array<Ts>::value...>::value,
+ "variant can not have an array type as an alternative.");
+
+ static_assert(lib::all<!std::is_reference<Ts>::value...>::value,
+ "variant can not have a reference type as an alternative.");
+
+ static_assert(lib::all<!std::is_void<Ts>::value...>::value,
+ "variant can not have a void type as an alternative.");
+
+ public:
+ template <
+ typename Front = lib::type_pack_element_t<0, Ts...>,
+ lib::enable_if_t<std::is_default_constructible<Front>::value, int> = 0>
+ inline constexpr variant() noexcept(
+ std::is_nothrow_default_constructible<Front>::value)
+ : impl_(in_place_index_t<0>{}) {}
+
+ variant(const variant &) = default;
+ variant(variant &&) = default;
+
+ template <
+ typename Arg,
+ typename Decayed = lib::decay_t<Arg>,
+ lib::enable_if_t<!std::is_same<Decayed, variant>::value, int> = 0,
+ lib::enable_if_t<!detail::is_in_place_index<Decayed>::value, int> = 0,
+ lib::enable_if_t<!detail::is_in_place_type<Decayed>::value, int> = 0,
+ std::size_t I = detail::best_match<Arg, Ts...>::value,
+ typename T = lib::type_pack_element_t<I, Ts...>,
+ lib::enable_if_t<std::is_constructible<T, Arg>::value, int> = 0>
+ inline constexpr variant(Arg &&arg) noexcept(
+ std::is_nothrow_constructible<T, Arg>::value)
+ : impl_(in_place_index_t<I>{}, lib::forward<Arg>(arg)) {}
+
+ template <
+ std::size_t I,
+ typename... Args,
+ typename T = lib::type_pack_element_t<I, Ts...>,
+ lib::enable_if_t<std::is_constructible<T, Args...>::value, int> = 0>
+ inline explicit constexpr variant(
+ in_place_index_t<I>,
+ Args &&... args) noexcept(std::is_nothrow_constructible<T,
+ Args...>::value)
+ : impl_(in_place_index_t<I>{}, lib::forward<Args>(args)...) {}
+
+ template <
+ std::size_t I,
+ typename Up,
+ typename... Args,
+ typename T = lib::type_pack_element_t<I, Ts...>,
+ lib::enable_if_t<std::is_constructible<T,
+ std::initializer_list<Up> &,
+ Args...>::value,
+ int> = 0>
+ inline explicit constexpr variant(
+ in_place_index_t<I>,
+ std::initializer_list<Up> il,
+ Args &&... args) noexcept(std::
+ is_nothrow_constructible<
+ T,
+ std::initializer_list<Up> &,
+ Args...>::value)
+ : impl_(in_place_index_t<I>{}, il, lib::forward<Args>(args)...) {}
+
+ template <
+ typename T,
+ typename... Args,
+ std::size_t I = detail::find_index_sfinae<T, Ts...>::value,
+ lib::enable_if_t<std::is_constructible<T, Args...>::value, int> = 0>
+ inline explicit constexpr variant(
+ in_place_type_t<T>,
+ Args &&... args) noexcept(std::is_nothrow_constructible<T,
+ Args...>::value)
+ : impl_(in_place_index_t<I>{}, lib::forward<Args>(args)...) {}
+
+ template <
+ typename T,
+ typename Up,
+ typename... Args,
+ std::size_t I = detail::find_index_sfinae<T, Ts...>::value,
+ lib::enable_if_t<std::is_constructible<T,
+ std::initializer_list<Up> &,
+ Args...>::value,
+ int> = 0>
+ inline explicit constexpr variant(
+ in_place_type_t<T>,
+ std::initializer_list<Up> il,
+ Args &&... args) noexcept(std::
+ is_nothrow_constructible<
+ T,
+ std::initializer_list<Up> &,
+ Args...>::value)
+ : impl_(in_place_index_t<I>{}, il, lib::forward<Args>(args)...) {}
+
+ ~variant() = default;
+
+ variant &operator=(const variant &) = default;
+ variant &operator=(variant &&) = default;
+
+ template <typename Arg,
+ lib::enable_if_t<!std::is_same<lib::decay_t<Arg>, variant>::value,
+ int> = 0,
+ std::size_t I = detail::best_match<Arg, Ts...>::value,
+ typename T = lib::type_pack_element_t<I, Ts...>,
+ lib::enable_if_t<(std::is_assignable<T &, Arg>::value &&
+ std::is_constructible<T, Arg>::value),
+ int> = 0>
+ inline variant &operator=(Arg &&arg) noexcept(
+ (std::is_nothrow_assignable<T &, Arg>::value &&
+ std::is_nothrow_constructible<T, Arg>::value)) {
+ impl_.template assign<I>(lib::forward<Arg>(arg));
+ return *this;
+ }
+
+ template <
+ std::size_t I,
+ typename... Args,
+ typename T = lib::type_pack_element_t<I, Ts...>,
+ lib::enable_if_t<std::is_constructible<T, Args...>::value, int> = 0>
+ inline T &emplace(Args &&... args) {
+ return impl_.template emplace<I>(lib::forward<Args>(args)...);
+ }
+
+ template <
+ std::size_t I,
+ typename Up,
+ typename... Args,
+ typename T = lib::type_pack_element_t<I, Ts...>,
+ lib::enable_if_t<std::is_constructible<T,
+ std::initializer_list<Up> &,
+ Args...>::value,
+ int> = 0>
+ inline T &emplace(std::initializer_list<Up> il, Args &&... args) {
+ return impl_.template emplace<I>(il, lib::forward<Args>(args)...);
+ }
+
+ template <
+ typename T,
+ typename... Args,
+ std::size_t I = detail::find_index_sfinae<T, Ts...>::value,
+ lib::enable_if_t<std::is_constructible<T, Args...>::value, int> = 0>
+ inline T &emplace(Args &&... args) {
+ return impl_.template emplace<I>(lib::forward<Args>(args)...);
+ }
+
+ template <
+ typename T,
+ typename Up,
+ typename... Args,
+ std::size_t I = detail::find_index_sfinae<T, Ts...>::value,
+ lib::enable_if_t<std::is_constructible<T,
+ std::initializer_list<Up> &,
+ Args...>::value,
+ int> = 0>
+ inline T &emplace(std::initializer_list<Up> il, Args &&... args) {
+ return impl_.template emplace<I>(il, lib::forward<Args>(args)...);
+ }
+
+ inline constexpr bool valueless_by_exception() const noexcept {
+ return impl_.valueless_by_exception();
+ }
+
+ inline constexpr std::size_t index() const noexcept {
+ return impl_.index();
+ }
+
+ template <bool Dummy = true,
+ lib::enable_if_t<
+ lib::all<Dummy,
+ (lib::dependent_type<std::is_move_constructible<Ts>,
+ Dummy>::value &&
+ lib::dependent_type<lib::is_swappable<Ts>,
+ Dummy>::value)...>::value,
+ int> = 0>
+ inline void swap(variant &that) noexcept(
+ lib::all<(std::is_nothrow_move_constructible<Ts>::value &&
+ lib::is_nothrow_swappable<Ts>::value)...>::value) {
+ impl_.swap(that.impl_);
+ }
+
+ private:
+ detail::impl<Ts...> impl_;
+
+ friend struct detail::access::variant;
+ friend struct detail::visitation::variant;
+ };
+
+ template <std::size_t I, typename... Ts>
+ inline constexpr bool holds_alternative(const variant<Ts...> &v) noexcept {
+ return v.index() == I;
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr bool holds_alternative(const variant<Ts...> &v) noexcept {
+ return holds_alternative<detail::find_index_checked<T, Ts...>::value>(v);
+ }
+
+ namespace detail {
+ template <std::size_t I, typename V>
+ struct generic_get_impl {
+ constexpr generic_get_impl(int) noexcept {}
+
+ constexpr AUTO_REFREF operator()(V &&v) const
+ AUTO_REFREF_RETURN(
+ access::variant::get_alt<I>(lib::forward<V>(v)).value)
+ };
+
+ template <std::size_t I, typename V>
+ inline constexpr AUTO_REFREF generic_get(V &&v)
+ AUTO_REFREF_RETURN(generic_get_impl<I, V>(
+ holds_alternative<I>(v) ? 0 : (throw_bad_variant_access(), 0))(
+ lib::forward<V>(v)))
+ } // namespace detail
+
+ template <std::size_t I, typename... Ts>
+ inline constexpr variant_alternative_t<I, variant<Ts...>> &get(
+ variant<Ts...> &v) {
+ return detail::generic_get<I>(v);
+ }
+
+ template <std::size_t I, typename... Ts>
+ inline constexpr variant_alternative_t<I, variant<Ts...>> &&get(
+ variant<Ts...> &&v) {
+ return detail::generic_get<I>(lib::move(v));
+ }
+
+ template <std::size_t I, typename... Ts>
+ inline constexpr const variant_alternative_t<I, variant<Ts...>> &get(
+ const variant<Ts...> &v) {
+ return detail::generic_get<I>(v);
+ }
+
+ template <std::size_t I, typename... Ts>
+ inline constexpr const variant_alternative_t<I, variant<Ts...>> &&get(
+ const variant<Ts...> &&v) {
+ return detail::generic_get<I>(lib::move(v));
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr T &get(variant<Ts...> &v) {
+ return get<detail::find_index_checked<T, Ts...>::value>(v);
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr T &&get(variant<Ts...> &&v) {
+ return get<detail::find_index_checked<T, Ts...>::value>(lib::move(v));
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr const T &get(const variant<Ts...> &v) {
+ return get<detail::find_index_checked<T, Ts...>::value>(v);
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr const T &&get(const variant<Ts...> &&v) {
+ return get<detail::find_index_checked<T, Ts...>::value>(lib::move(v));
+ }
+
+ namespace detail {
+
+ template <std::size_t I, typename V>
+ inline constexpr /* auto * */ AUTO generic_get_if(V *v) noexcept
+ AUTO_RETURN(v && holds_alternative<I>(*v)
+ ? lib::addressof(access::variant::get_alt<I>(*v).value)
+ : nullptr)
+
+ } // namespace detail
+
+ template <std::size_t I, typename... Ts>
+ inline constexpr lib::add_pointer_t<variant_alternative_t<I, variant<Ts...>>>
+ get_if(variant<Ts...> *v) noexcept {
+ return detail::generic_get_if<I>(v);
+ }
+
+ template <std::size_t I, typename... Ts>
+ inline constexpr lib::add_pointer_t<
+ const variant_alternative_t<I, variant<Ts...>>>
+ get_if(const variant<Ts...> *v) noexcept {
+ return detail::generic_get_if<I>(v);
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr lib::add_pointer_t<T>
+ get_if(variant<Ts...> *v) noexcept {
+ return get_if<detail::find_index_checked<T, Ts...>::value>(v);
+ }
+
+ template <typename T, typename... Ts>
+ inline constexpr lib::add_pointer_t<const T>
+ get_if(const variant<Ts...> *v) noexcept {
+ return get_if<detail::find_index_checked<T, Ts...>::value>(v);
+ }
+
+ namespace detail {
+ template <typename RelOp>
+ struct convert_to_bool {
+ template <typename Lhs, typename Rhs>
+ inline constexpr bool operator()(Lhs &&lhs, Rhs &&rhs) const {
+ static_assert(std::is_convertible<lib::invoke_result_t<RelOp, Lhs, Rhs>,
+ bool>::value,
+ "relational operators must return a type"
+ " implicitly convertible to bool");
+ return lib::invoke(
+ RelOp{}, lib::forward<Lhs>(lhs), lib::forward<Rhs>(rhs));
+ }
+ };
+ } // namespace detail
+
+ template <typename... Ts>
+ inline constexpr bool operator==(const variant<Ts...> &lhs,
+ const variant<Ts...> &rhs) {
+ using detail::visitation::variant;
+ using equal_to = detail::convert_to_bool<lib::equal_to>;
+#ifdef TF_CPP14_CONSTEXPR
+ if (lhs.index() != rhs.index()) return false;
+ if (lhs.valueless_by_exception()) return true;
+ return variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs);
+#else
+ return lhs.index() == rhs.index() &&
+ (lhs.valueless_by_exception() ||
+ variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs));
+#endif
+ }
+
+ template <typename... Ts>
+ inline constexpr bool operator!=(const variant<Ts...> &lhs,
+ const variant<Ts...> &rhs) {
+ using detail::visitation::variant;
+ using not_equal_to = detail::convert_to_bool<lib::not_equal_to>;
+#ifdef TF_CPP14_CONSTEXPR
+ if (lhs.index() != rhs.index()) return true;
+ if (lhs.valueless_by_exception()) return false;
+ return variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs);
+#else
+ return lhs.index() != rhs.index() ||
+ (!lhs.valueless_by_exception() &&
+ variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs));
+#endif
+ }
+
+ template <typename... Ts>
+ inline constexpr bool operator<(const variant<Ts...> &lhs,
+ const variant<Ts...> &rhs) {
+ using detail::visitation::variant;
+ using less = detail::convert_to_bool<lib::less>;
+#ifdef TF_CPP14_CONSTEXPR
+ if (rhs.valueless_by_exception()) return false;
+ if (lhs.valueless_by_exception()) return true;
+ if (lhs.index() < rhs.index()) return true;
+ if (lhs.index() > rhs.index()) return false;
+ return variant::visit_value_at(lhs.index(), less{}, lhs, rhs);
+#else
+ return !rhs.valueless_by_exception() &&
+ (lhs.valueless_by_exception() || lhs.index() < rhs.index() ||
+ (lhs.index() == rhs.index() &&
+ variant::visit_value_at(lhs.index(), less{}, lhs, rhs)));
+#endif
+ }
+
+ template <typename... Ts>
+ inline constexpr bool operator>(const variant<Ts...> &lhs,
+ const variant<Ts...> &rhs) {
+ using detail::visitation::variant;
+ using greater = detail::convert_to_bool<lib::greater>;
+#ifdef TF_CPP14_CONSTEXPR
+ if (lhs.valueless_by_exception()) return false;
+ if (rhs.valueless_by_exception()) return true;
+ if (lhs.index() > rhs.index()) return true;
+ if (lhs.index() < rhs.index()) return false;
+ return variant::visit_value_at(lhs.index(), greater{}, lhs, rhs);
+#else
+ return !lhs.valueless_by_exception() &&
+ (rhs.valueless_by_exception() || lhs.index() > rhs.index() ||
+ (lhs.index() == rhs.index() &&
+ variant::visit_value_at(lhs.index(), greater{}, lhs, rhs)));
+#endif
+ }
+
+ template <typename... Ts>
+ inline constexpr bool operator<=(const variant<Ts...> &lhs,
+ const variant<Ts...> &rhs) {
+ using detail::visitation::variant;
+ using less_equal = detail::convert_to_bool<lib::less_equal>;
+#ifdef TF_CPP14_CONSTEXPR
+ if (lhs.valueless_by_exception()) return true;
+ if (rhs.valueless_by_exception()) return false;
+ if (lhs.index() < rhs.index()) return true;
+ if (lhs.index() > rhs.index()) return false;
+ return variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs);
+#else
+ return lhs.valueless_by_exception() ||
+ (!rhs.valueless_by_exception() &&
+ (lhs.index() < rhs.index() ||
+ (lhs.index() == rhs.index() &&
+ variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs))));
+#endif
+ }
+
+ template <typename... Ts>
+ inline constexpr bool operator>=(const variant<Ts...> &lhs,
+ const variant<Ts...> &rhs) {
+ using detail::visitation::variant;
+ using greater_equal = detail::convert_to_bool<lib::greater_equal>;
+#ifdef TF_CPP14_CONSTEXPR
+ if (rhs.valueless_by_exception()) return true;
+ if (lhs.valueless_by_exception()) return false;
+ if (lhs.index() > rhs.index()) return true;
+ if (lhs.index() < rhs.index()) return false;
+ return variant::visit_value_at(lhs.index(), greater_equal{}, lhs, rhs);
+#else
+ return rhs.valueless_by_exception() ||
+ (!lhs.valueless_by_exception() &&
+ (lhs.index() > rhs.index() ||
+ (lhs.index() == rhs.index() &&
+ variant::visit_value_at(
+ lhs.index(), greater_equal{}, lhs, rhs))));
+#endif
+ }
+
+ struct monostate {};
+
+ inline constexpr bool operator<(monostate, monostate) noexcept {
+ return false;
+ }
+
+ inline constexpr bool operator>(monostate, monostate) noexcept {
+ return false;
+ }
+
+ inline constexpr bool operator<=(monostate, monostate) noexcept {
+ return true;
+ }
+
+ inline constexpr bool operator>=(monostate, monostate) noexcept {
+ return true;
+ }
+
+ inline constexpr bool operator==(monostate, monostate) noexcept {
+ return true;
+ }
+
+ inline constexpr bool operator!=(monostate, monostate) noexcept {
+ return false;
+ }
+
+#ifdef TF_CPP14_CONSTEXPR
+ namespace detail {
+
+ inline constexpr bool any(std::initializer_list<bool> bs) {
+ for (bool b : bs) {
+ if (b) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ } // namespace detail
+
+ template <typename Visitor, typename... Vs>
+ inline constexpr decltype(auto) visit(Visitor &&visitor, Vs &&... vs) {
+ return (!detail::any({vs.valueless_by_exception()...})
+ ? (void)0
+ : throw_bad_variant_access()),
+ detail::visitation::variant::visit_value(
+ lib::forward<Visitor>(visitor), lib::forward<Vs>(vs)...);
+ }
+#else
+ namespace detail {
+
+ template <std::size_t N>
+ inline constexpr bool all_impl(const lib::array<bool, N> &bs,
+ std::size_t idx) {
+ return idx >= N || (bs[idx] && all_impl(bs, idx + 1));
+ }
+
+ template <std::size_t N>
+ inline constexpr bool all(const lib::array<bool, N> &bs) {
+ return all_impl(bs, 0);
+ }
+
+ } // namespace detail
+
+ template <typename Visitor, typename... Vs>
+ inline constexpr DECLTYPE_AUTO visit(Visitor &&visitor, Vs &&... vs)
+ DECLTYPE_AUTO_RETURN(
+ (detail::all(
+ lib::array<bool, sizeof...(Vs)>{{!vs.valueless_by_exception()...}})
+ ? (void)0
+ : throw_bad_variant_access()),
+ detail::visitation::variant::visit_value(lib::forward<Visitor>(visitor),
+ lib::forward<Vs>(vs)...))
+#endif
+
+ template <typename... Ts>
+ inline auto swap(variant<Ts...> &lhs,
+ variant<Ts...> &rhs) noexcept(noexcept(lhs.swap(rhs)))
+ -> decltype(lhs.swap(rhs)) {
+ lhs.swap(rhs);
+ }
+
+ namespace detail {
+
+ template <typename T, typename...>
+ using enabled_type = T;
+
+ namespace hash {
+
+ template <typename H, typename K>
+ constexpr bool meets_requirements() noexcept {
+ return std::is_copy_constructible<H>::value &&
+ std::is_move_constructible<H>::value &&
+ lib::is_invocable_r<std::size_t, H, const K &>::value;
+ }
+
+ template <typename K>
+ constexpr bool is_enabled() noexcept {
+ using H = std::hash<K>;
+ return meets_requirements<H, K>() &&
+ std::is_default_constructible<H>::value &&
+ std::is_copy_assignable<H>::value &&
+ std::is_move_assignable<H>::value;
+ }
+
+ } // namespace hash
+
+ } // namespace detail
+
+#undef AUTO
+#undef AUTO_RETURN
+
+#undef AUTO_REFREF
+#undef AUTO_REFREF_RETURN
+
+#undef DECLTYPE_AUTO
+#undef DECLTYPE_AUTO_RETURN
+
+}} // namespace tf::nstd
+
+namespace std {
+
+ template <typename... Ts>
+ struct hash<tf::nstd::detail::enabled_type<
+ tf::nstd::variant<Ts...>,
+ tf::nstd::lib::enable_if_t<tf::nstd::lib::all<tf::nstd::detail::hash::is_enabled<
+ tf::nstd::lib::remove_const_t<Ts>>()...>::value>>> {
+ using argument_type = tf::nstd::variant<Ts...>;
+ using result_type = std::size_t;
+
+ inline result_type operator()(const argument_type &v) const {
+ using tf::nstd::detail::visitation::variant;
+ std::size_t result =
+ v.valueless_by_exception()
+ ? 299792458 // Random value chosen by the universe upon creation
+ : variant::visit_alt(
+#ifdef TF_GENERIC_LAMBDAS
+ [](const auto &alt) {
+ using alt_type = tf::nstd::lib::decay_t<decltype(alt)>;
+ using value_type = tf::nstd::lib::remove_const_t<
+ typename alt_type::value_type>;
+ return hash<value_type>{}(alt.value);
+ }
+#else
+ hasher{}
+#endif
+ ,
+ v);
+ return hash_combine(result, hash<std::size_t>{}(v.index()));
+ }
+
+ private:
+#ifndef TF_GENERIC_LAMBDAS
+ struct hasher {
+ template <typename Alt>
+ inline std::size_t operator()(const Alt &alt) const {
+ using alt_type = tf::nstd::lib::decay_t<Alt>;
+ using value_type =
+ tf::nstd::lib::remove_const_t<typename alt_type::value_type>;
+ return hash<value_type>{}(alt.value);
+ }
+ };
+#endif
+
+ static std::size_t hash_combine(std::size_t lhs, std::size_t rhs) {
+ return lhs ^= rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2);
+ }
+ };
+
+ template <>
+ struct hash<tf::nstd::monostate> {
+ using argument_type = tf::nstd::monostate;
+ using result_type = std::size_t;
+
+ inline result_type operator()(const argument_type &) const noexcept {
+ return 66740831; // return a fundamentally attractive random value.
+ }
+ };
+
+} // namespace std
+
+
+
--- /dev/null
+#pragma once
+
+#include "core/executor.hpp"
+
+namespace tf {
+
+
+} // end of namespace tf. ---------------------------------------------------
+
+
+
+
+
--- /dev/null
+// 2020/03/13 - modified by Tsung-Wei Huang
+// - fixed bug in aligning memory
+//
+// 2020/02/02 - modified by Tsung-Wei Huang
+// - new implementation motivated by Hoard
+//
+// 2019/07/10 - modified by Tsung-Wei Huang
+// - replace raw pointer with smart pointer
+//
+// 2019/06/13 - created by Tsung-Wei Huang
+// - implemented an object pool class
+
+#pragma once
+
+#include <thread>
+#include <atomic>
+#include <mutex>
+#include <vector>
+#include <cassert>
+#include <cstddef>
+
+namespace tf {
+
+#define TF_ENABLE_POOLABLE_ON_THIS \
+ template <typename T, size_t S> friend class ObjectPool; \
+ void* _object_pool_block;
+
+// Class: ObjectPool
+//
+// The class implements an efficient thread-safe object pool motivated
+// by the Hoard memory allocator algorithm.
+// Different from the normal memory allocator, object pool allocates
+// only one object at a time.
+//
+// Internall, we use the following variables to maintain blocks and heaps:
+// X: size in byte of a item slot
+// M: number of items per block
+// F: emptiness threshold
+// B: number of bins per local heap (bin[B-1] is the full list)
+// W: number of items per bin
+// K: shrinkness constant
+//
+// Example scenario 1:
+// M = 30
+// F = 4
+// W = (30+4-1)/4 = 8
+//
+// b0: 0, 1, 2, 3, 4, 5, 6, 7
+// b1: 8, 9, 10, 11, 12, 13, 14, 15
+// b2: 16, 17, 18, 19, 20, 21, 22, 23
+// b3: 24, 25, 26, 27, 28, 29
+// b4: 30 (anything equal to M)
+//
+// Example scenario 2:
+// M = 32
+// F = 4
+// W = (32+4-1)/4 = 8
+// b0: 0, 1, 2, 3, 4, 5, 6, 7
+// b1: 8, 9, 10, 11, 12, 13, 14, 15
+// b2: 16, 17, 18, 19, 20, 21, 22, 23
+// b3: 24, 25, 26, 27, 28, 29, 30, 31
+// b4: 32 (anything equal to M)
+//
+template <typename T, size_t S = 65536>
+class ObjectPool {
+
+ // the data column must be sufficient to hold the pointer in freelist
+ constexpr static size_t X = std::max(sizeof(T*), sizeof(T));
+ //constexpr static size_t X = sizeof(long double) + std::max(sizeof(T*), sizeof(T));
+ //constexpr static size_t M = (S - offsetof(Block, data)) / X;
+ constexpr static size_t M = S / X;
+ constexpr static size_t F = 4;
+ constexpr static size_t B = F + 1;
+ constexpr static size_t W = (M + F - 1) / F;
+ constexpr static size_t K = 4;
+
+ static_assert(
+ S && (!(S & (S-1))), "block size S must be a power of two"
+ );
+
+ static_assert(
+ M >= 128, "block size S must be larger enough to pool at least 128 objects"
+ );
+
+ struct Blocklist {
+ Blocklist* prev;
+ Blocklist* next;
+ };
+
+ class GlobalHeap {
+ friend class ObjectPool;
+ std::mutex mutex;
+ Blocklist list;
+ };
+
+ class LocalHeap {
+ friend class ObjectPool;
+ std::mutex mutex;
+ Blocklist lists[B];
+ size_t u {0};
+ size_t a {0};
+ };
+
+ struct Block {
+ LocalHeap* heap;
+ Blocklist list_node;
+ size_t i;
+ size_t u;
+ T* top;
+ // long double padding;
+ char data[S];
+ };
+
+ public:
+
+ /**
+ @brief constructs an object pool from a number of anticipated threads
+ */
+ explicit ObjectPool(unsigned = std::thread::hardware_concurrency());
+
+ /**
+ @brief destructs the object pool
+ */
+ ~ObjectPool();
+
+ /**
+ @brief acquires a pointer to a object constructed from a given argument list
+ */
+ template <typename... ArgsT>
+ T* animate(ArgsT&&... args);
+
+ /**
+ @brief recycles a object pointed by @c ptr and destroys it
+ */
+ void recycle(T* ptr);
+
+ size_t num_bins_per_local_heap() const;
+ size_t num_objects_per_bin() const;
+ size_t num_objects_per_block() const;
+ size_t num_available_objects() const;
+ size_t num_allocated_objects() const;
+ size_t capacity() const;
+ size_t num_local_heaps() const;
+ size_t num_global_heaps() const;
+ size_t num_heaps() const;
+
+ float emptiness_threshold() const;
+
+ private:
+
+ const size_t _lheap_mask;
+
+ GlobalHeap _gheap;
+
+ std::vector<LocalHeap> _lheaps;
+
+ LocalHeap& _this_heap();
+
+ constexpr unsigned _next_power_of_two(unsigned n) const;
+
+ template <class P, class Q>
+ constexpr size_t _offset_in_class(const Q P::*member) const;
+
+ template <class P, class Q>
+ constexpr P* _parent_class_of(Q*, const Q P::*member);
+
+ template <class P, class Q>
+ constexpr P* _parent_class_of(const Q*, const Q P::*member) const;
+
+ constexpr Block* _block_of(Blocklist*);
+ constexpr Block* _block_of(const Blocklist*) const;
+
+ size_t _bin(size_t) const;
+
+ T* _allocate(Block*);
+
+ void _deallocate(Block*, T*);
+ void _blocklist_init_head(Blocklist*);
+ void _blocklist_add_impl(Blocklist*, Blocklist*, Blocklist*);
+ void _blocklist_push_front(Blocklist*, Blocklist*);
+ void _blocklist_push_back(Blocklist*, Blocklist*);
+ void _blocklist_del_impl(Blocklist*, Blocklist*);
+ void _blocklist_del(Blocklist*);
+ void _blocklist_replace(Blocklist*, Blocklist*);
+ void _blocklist_move_front(Blocklist*, Blocklist*);
+ void _blocklist_move_back(Blocklist*, Blocklist*);
+ bool _blocklist_is_first(const Blocklist*, const Blocklist*);
+ bool _blocklist_is_last(const Blocklist*, const Blocklist*);
+ bool _blocklist_is_empty(const Blocklist*);
+ bool _blocklist_is_singular(const Blocklist*);
+
+ template <typename C>
+ void _for_each_block_safe(Blocklist*, C&&);
+
+ template <typename C>
+ void _for_each_block(Blocklist*, C&&);
+
+};
+
+// ----------------------------------------------------------------------------
+// ObjectPool definition
+// ----------------------------------------------------------------------------
+
+// Constructor
+template <typename T, size_t S>
+ObjectPool<T, S>::ObjectPool(unsigned t) :
+ //_heap_mask {(_next_power_of_two(t) << 1) - 1u},
+ //_heap_mask { _next_power_of_two(t<<1) - 1u },
+ //_heap_mask {(t << 1) - 1},
+ _lheap_mask { _next_power_of_two((t+1) << 1) - 1 },
+ _lheaps { _lheap_mask + 1 } {
+
+ _blocklist_init_head(&_gheap.list);
+
+ for(auto& h : _lheaps) {
+ for(size_t i=0; i<B; ++i) {
+ _blocklist_init_head(&h.lists[i]);
+ }
+ }
+}
+
+// Destructor
+template <typename T, size_t S>
+ObjectPool<T, S>::~ObjectPool() {
+
+ // clear local heaps
+ for(auto& h : _lheaps) {
+ for(size_t i=0; i<B; ++i) {
+ _for_each_block_safe(&h.lists[i], [] (Block* b) {
+ std::free(b);
+ });
+ }
+ }
+
+ // clear global heap
+ _for_each_block_safe(&_gheap.list, [] (Block* b) {
+ std::free(b);
+ });
+}
+
+// Function: num_bins_per_local_heap
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_bins_per_local_heap() const {
+ return B;
+}
+
+// Function: num_objects_per_bin
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_objects_per_bin() const {
+ return W;
+}
+
+// Function: num_objects_per_block
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_objects_per_block() const {
+ return M;
+}
+
+// Function: emptiness_threshold
+template <typename T, size_t S>
+float ObjectPool<T, S>::emptiness_threshold() const {
+ return 1.0f/F;
+}
+
+// Function: num_global_heaps
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_global_heaps() const {
+ return 1;
+}
+
+// Function: num_lheaps
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_local_heaps() const {
+ return _lheaps.size();
+}
+
+// Function: num_heaps
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_heaps() const {
+ return _lheaps.size() + 1;
+}
+
+// Function: capacity
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::capacity() const {
+
+ size_t n = 0;
+
+ // global heap
+ for(auto p=_gheap.list.next; p!=&_gheap.list; p=p->next) {
+ n += M;
+ };
+
+ // local heap
+ for(auto& h : _lheaps) {
+ n += h.a;
+ }
+
+ return n;
+}
+
+// Function: num_available_objects
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_available_objects() const {
+
+ size_t n = 0;
+
+ // global heap
+ for(auto p=_gheap.list.next; p!=&_gheap.list; p=p->next) {
+ n += (M - _block_of(p)->u);
+ };
+
+ // local heap
+ for(auto& h : _lheaps) {
+ n += (h.a - h.u);
+ }
+ return n;
+}
+
+// Function: num_allocated_objects
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::num_allocated_objects() const {
+
+ size_t n = 0;
+
+ // global heap
+ for(auto p=_gheap.list.next; p!=&_gheap.list; p=p->next) {
+ n += _block_of(p)->u;
+ };
+
+ // local heap
+ for(auto& h : _lheaps) {
+ n += h.u;
+ }
+ return n;
+}
+
+// Function: _bin
+template <typename T, size_t S>
+size_t ObjectPool<T, S>::_bin(size_t u) const {
+ return u == M ? F : u/W;
+}
+
+// Function: _offset_in_class
+template <typename T, size_t S>
+template <class P, class Q>
+constexpr size_t ObjectPool<T, S>::_offset_in_class(
+ const Q P::*member) const {
+ return (size_t) &( reinterpret_cast<P*>(0)->*member);
+}
+
+// C macro: parent_class_of(list_pointer, Block, list)
+// C++: parent_class_of(list_pointer, &Block::list)
+template <typename T, size_t S>
+template <class P, class Q>
+constexpr P* ObjectPool<T, S>::_parent_class_of(
+ Q* ptr, const Q P::*member
+) {
+ return (P*)( (char*)ptr - _offset_in_class(member));
+}
+
+// Function: _parent_class_of
+template <typename T, size_t S>
+template <class P, class Q>
+constexpr P* ObjectPool<T, S>::_parent_class_of(
+ const Q* ptr, const Q P::*member
+) const {
+ return (P*)( (char*)ptr - _offset_in_class(member));
+}
+
+// Function: _block_of
+template <typename T, size_t S>
+constexpr typename ObjectPool<T, S>::Block*
+ObjectPool<T, S>::_block_of(Blocklist* list) {
+ return _parent_class_of(list, &Block::list_node);
+}
+
+// Function: _block_of
+template <typename T, size_t S>
+constexpr typename ObjectPool<T, S>::Block*
+ObjectPool<T, S>::_block_of(const Blocklist* list) const {
+ return _parent_class_of(list, &Block::list_node);
+}
+
+// Procedure: initialize a list head
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_init_head(Blocklist *list) {
+ list->next = list;
+ list->prev = list;
+}
+
+// Procedure: _blocklist_add_impl
+// Insert a new entry between two known consecutive entries.
+//
+// This is only for internal list manipulation where we know
+// the prev/next entries already!
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_add_impl(
+ Blocklist *curr, Blocklist *prev, Blocklist *next
+) {
+ next->prev = curr;
+ curr->next = next;
+ curr->prev = prev;
+ prev->next = curr;
+}
+
+// list_push_front - add a new entry
+// @curr: curr entry to be added
+// @head: list head to add it after
+//
+// Insert a new entry after the specified head.
+// This is good for implementing stacks.
+//
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_push_front(
+ Blocklist *curr, Blocklist *head
+) {
+ _blocklist_add_impl(curr, head, head->next);
+}
+
+// list_add_tail - add a new entry
+// @curr: curr entry to be added
+// @head: list head to add it before
+//
+// Insert a new entry before the specified head.
+// This is useful for implementing queues.
+//
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_push_back(
+ Blocklist *curr, Blocklist *head
+) {
+ _blocklist_add_impl(curr, head->prev, head);
+}
+
+// Delete a list entry by making the prev/next entries
+// point to each other.
+//
+// This is only for internal list manipulation where we know
+// the prev/next entries already!
+//
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_del_impl(
+ Blocklist * prev, Blocklist * next
+) {
+ next->prev = prev;
+ prev->next = next;
+}
+
+// _blocklist_del - deletes entry from list.
+// @entry: the element to delete from the list.
+// Note: list_empty() on entry does not return true after this, the entry is
+// in an undefined state.
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_del(Blocklist *entry) {
+ _blocklist_del_impl(entry->prev, entry->next);
+ entry->next = nullptr;
+ entry->prev = nullptr;
+}
+
+// list_replace - replace old entry by new one
+// @old : the element to be replaced
+// @curr : the new element to insert
+//
+// If @old was empty, it will be overwritten.
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_replace(
+ Blocklist *old, Blocklist *curr
+) {
+ curr->next = old->next;
+ curr->next->prev = curr;
+ curr->prev = old->prev;
+ curr->prev->next = curr;
+}
+
+// list_move - delete from one list and add as another's head
+// @list: the entry to move
+// @head: the head that will precede our entry
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_move_front(
+ Blocklist *list, Blocklist *head
+) {
+ _blocklist_del_impl(list->prev, list->next);
+ _blocklist_push_front(list, head);
+}
+
+// list_move_tail - delete from one list and add as another's tail
+// @list: the entry to move
+// @head: the head that will follow our entry
+template <typename T, size_t S>
+void ObjectPool<T, S>::_blocklist_move_back(
+ Blocklist *list, Blocklist *head
+) {
+ _blocklist_del_impl(list->prev, list->next);
+ _blocklist_push_back(list, head);
+}
+
+// list_is_first - tests whether @list is the last entry in list @head
+// @list: the entry to test
+// @head: the head of the list
+template <typename T, size_t S>
+bool ObjectPool<T, S>::_blocklist_is_first(
+ const Blocklist *list, const Blocklist *head
+) {
+ return list->prev == head;
+}
+
+// list_is_last - tests whether @list is the last entry in list @head
+// @list: the entry to test
+// @head: the head of the list
+template <typename T, size_t S>
+bool ObjectPool<T, S>::_blocklist_is_last(
+ const Blocklist *list, const Blocklist *head
+) {
+ return list->next == head;
+}
+
+// list_empty - tests whether a list is empty
+// @head: the list to test.
+template <typename T, size_t S>
+bool ObjectPool<T, S>::_blocklist_is_empty(const Blocklist *head) {
+ return head->next == head;
+}
+
+// list_is_singular - tests whether a list has just one entry.
+// @head: the list to test.
+template <typename T, size_t S>
+bool ObjectPool<T, S>::_blocklist_is_singular(
+ const Blocklist *head
+) {
+ return !_blocklist_is_empty(head) && (head->next == head->prev);
+}
+
+// Procedure: _for_each_block
+template <typename T, size_t S>
+template <typename C>
+void ObjectPool<T, S>::_for_each_block(Blocklist* head, C&& c) {
+ Blocklist* p;
+ for(p=head->next; p!=head; p=p->next) {
+ c(_block_of(p));
+ }
+}
+
+// Procedure: _for_each_block_safe
+// Iterate each item of a list - safe to free
+template <typename T, size_t S>
+template <typename C>
+void ObjectPool<T, S>::_for_each_block_safe(Blocklist* head, C&& c) {
+ Blocklist* p;
+ Blocklist* t;
+ for(p=head->next, t=p->next; p!=head; p=t, t=p->next) {
+ c(_block_of(p));
+ }
+}
+
+// Function: _allocate
+// allocate a spot from the block
+template <typename T, size_t S>
+T* ObjectPool<T, S>::_allocate(Block* s) {
+ if(s->top == nullptr) {
+ return reinterpret_cast<T*>(s->data + s->i++ * X);
+ }
+ else {
+ T* retval = s->top;
+ s->top = *(reinterpret_cast<T**>(s->top));
+ return retval;
+ }
+}
+
+// Procedure: _deallocate
+template <typename T, size_t S>
+void ObjectPool<T, S>::_deallocate(Block* s, T* ptr) {
+ *(reinterpret_cast<T**>(ptr)) = s->top;
+ s->top = ptr;
+}
+
+// Function: allocate
+template <typename T, size_t S>
+template <typename... ArgsT>
+T* ObjectPool<T, S>::animate(ArgsT&&... args) {
+
+ //std::cout << "construct a new item\n";
+
+ // my logically mapped heap
+ LocalHeap& h = _this_heap();
+
+ Block* s {nullptr};
+
+ h.mutex.lock();
+
+ // scan the list of superblocks from most full to least
+ int f = static_cast<int>(F-1);
+ for(; f>=0; f--) {
+ if(!_blocklist_is_empty(&h.lists[f])) {
+ s = _block_of(h.lists[f].next);
+ break;
+ }
+ }
+
+ // no superblock found
+ if(f == -1) {
+
+ // check heap 0 for a superblock
+ _gheap.mutex.lock();
+ if(!_blocklist_is_empty(&_gheap.list)) {
+
+ s = _block_of(_gheap.list.next);
+
+ //printf("get a superblock from global heap %lu\n", s->u);
+ assert(s->u < M && s->heap == nullptr);
+ f = static_cast<int>(_bin(s->u + 1));
+
+ _blocklist_move_front(&s->list_node, &h.lists[f]);
+
+ s->heap = &h; // must be within the global heap lock
+ _gheap.mutex.unlock();
+
+ h.u = h.u + s->u;
+ h.a = h.a + M;
+ }
+ // create a new block
+ else {
+ //printf("create a new superblock\n");
+ _gheap.mutex.unlock();
+ f = 0;
+ s = static_cast<Block*>(std::malloc(sizeof(Block)));
+
+ if(s == nullptr) {
+ throw std::bad_alloc();
+ }
+
+ s->heap = &h;
+ s->i = 0;
+ s->u = 0;
+ s->top = nullptr;
+
+ _blocklist_push_front(&s->list_node, &h.lists[f]);
+
+ h.a = h.a + M;
+ }
+ }
+
+ // the superblock must have at least one space
+ //assert(s->u < M);
+ //printf("%lu %lu %lu\n", h.u, h.a, s->u);
+ //assert(h.u < h.a);
+
+ h.u = h.u + 1;
+ s->u = s->u + 1;
+
+ // take one item from the superblock
+ T* mem = _allocate(s);
+
+ int b = static_cast<int>(_bin(s->u));
+
+ if(b != f) {
+ //printf("move superblock from list[%d] to list[%d]\n", f, b);
+ _blocklist_move_front(&s->list_node, &h.lists[b]);
+ }
+
+ //std::cout << "s.i " << s->i << '\n'
+ // << "s.u " << s->u << '\n'
+ // << "h.u " << h.u << '\n'
+ // << "h.a " << h.a << '\n';
+
+ h.mutex.unlock();
+
+ //printf("allocate %p (s=%p)\n", mem, s);
+
+ new (mem) T(std::forward<ArgsT>(args)...);
+
+ mem->_object_pool_block = s;
+
+ return mem;
+}
+
+// Function: destruct
+template <typename T, size_t S>
+void ObjectPool<T, S>::recycle(T* mem) {
+
+ //Block* s = *reinterpret_cast<Block**>(
+ // reinterpret_cast<char*>(mem) - sizeof(Block**)
+ //);
+
+ //Block* s= *(reinterpret_cast<Block**>(mem) - O); // (mem) - 1
+
+ Block* s = static_cast<Block*>(mem->_object_pool_block);
+
+ mem->~T();
+
+ //printf("deallocate %p (s=%p) M=%lu W=%lu X=%lu\n", mem, s, M, W, X);
+
+ // here we need a loop because when we lock the heap,
+ // other threads may have removed the superblock to another heap
+ bool sync = false;
+
+ do {
+ auto h = s->heap;
+
+ // the block is in global heap
+ if(h == nullptr) {
+ std::lock_guard<std::mutex> glock(_gheap.mutex);
+ if(s->heap == h) {
+ sync = true;
+ _deallocate(s, mem);
+ s->u = s->u - 1;
+ }
+ }
+ else {
+ std::lock_guard<std::mutex> llock(h->mutex);
+ if(s->heap == h) {
+ sync = true;
+ // deallocate the item from the superblock
+ size_t f = _bin(s->u);
+ _deallocate(s, mem);
+ s->u = s->u - 1;
+ h->u = h->u - 1;
+
+ size_t b = _bin(s->u);
+
+ if(b != f) {
+ //printf("move superblock from list[%d] to list[%d]\n", f, b);
+ _blocklist_move_front(&s->list_node, &h->lists[b]);
+ }
+
+ // transfer a mostly-empty superblock to global heap
+ if((h->u + K*M < h->a) && (h->u < ((F-1) * h->a / F))) {
+ for(size_t i=0; i<F; i++) {
+ if(!_blocklist_is_empty(&h->lists[i])) {
+ Block* x = _block_of(h->lists[i].next);
+ //printf("transfer a block (x.u=%lu/x.i=%lu) to the global heap\n", x->u, x->i);
+ assert(h->u > x->u && h->a > M);
+ h->u = h->u - x->u;
+ h->a = h->a - M;
+ x->heap = nullptr;
+ std::lock_guard<std::mutex> glock(_gheap.mutex);
+ _blocklist_move_front(&x->list_node, &_gheap.list);
+ break;
+ }
+ }
+ }
+ }
+ }
+ } while(!sync);
+
+ //std::cout << "s.i " << s->i << '\n'
+ // << "s.u " << s->u << '\n';
+}
+
+// Function: _this_heap
+template <typename T, size_t S>
+typename ObjectPool<T, S>::LocalHeap&
+ObjectPool<T, S>::_this_heap() {
+ // here we don't use thread local since object pool might be
+ // created and destroyed multiple times
+ thread_local auto hv = std::hash<std::thread::id>()(std::this_thread::get_id());
+ return _lheaps[hv & _lheap_mask];
+
+ //return _lheaps[
+ // std::hash<std::thread::id>()(std::this_thread::get_id()) & _lheap_mask
+ //];
+}
+
+// Function: _next_power_of_two
+template <typename T, size_t S>
+constexpr unsigned ObjectPool<T, S>::_next_power_of_two(unsigned n) const {
+ n--;
+ n |= n >> 1;
+ n |= n >> 2;
+ n |= n >> 4;
+ n |= n >> 8;
+ n |= n >> 16;
+ n++;
+ return n;
+}
+
+} // end namespace tf --------------------------------------------------------
--- /dev/null
+#include <cstdlib>
+#include <cstdio>
+#include <string>
+
+namespace tf {
+
+// Function: get_env
+inline std::string get_env(const std::string& str) {
+#ifdef _MSC_VER
+ char *ptr = nullptr;
+ size_t len = 0;
+
+ if(_dupenv_s(&ptr, &len, str.c_str()) == 0 && ptr != nullptr) {
+ std::string res(ptr, len);
+ free(ptr);
+ return res;
+ }
+ return "";
+
+#else
+ auto ptr = std::getenv(str.c_str());
+ return ptr ? ptr : "";
+#endif
+}
+
+
+} // end of namespace tf -----------------------------------------------------
--- /dev/null
+#pragma once
+
+#include <cstring>
+#include <cstddef>
+#include <memory>
+#include <type_traits>
+#include <limits>
+
+namespace tf {
+
+// Class: PassiveVector
+// A vector storing only passive data structure (PDS) or POD data type.
+template <typename T, size_t S = 4, typename A = std::allocator<T>>
+class PassiveVector {
+
+ static_assert(
+ std::is_trivial<T>::value && std::is_standard_layout<T>::value,
+ "must be a plain old data type"
+ );
+
+ public:
+
+ typedef T value_type;
+ typedef T & reference;
+ typedef const T & const_reference;
+ typedef T * pointer;
+ typedef const T * const_pointer;
+ typedef T * iterator;
+ typedef const T * const_iterator;
+ typedef std::reverse_iterator<iterator> reverse_iterator;
+ typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+ typedef ptrdiff_t difference_type;
+ typedef size_t size_type;
+
+ PassiveVector() noexcept :
+ _data {reinterpret_cast<pointer>(_stack)},
+ _num {0},
+ _cap {S} {
+ }
+
+ explicit PassiveVector(size_type n) : _num {n} {
+
+ // need to place on heap
+ if(n > S) {
+ _cap = n << 2;
+ _data = _allocator.allocate(_cap);
+ }
+ // stack
+ else {
+ _cap = S;
+ _data = reinterpret_cast<pointer>(_stack);
+ }
+
+ }
+
+ PassiveVector(const PassiveVector& rhs) : _num {rhs._num} {
+
+ // heap
+ if(rhs._num > S) {
+ _cap = rhs._cap;
+ _data = _allocator.allocate(rhs._cap);
+ }
+ else {
+ _cap = S;
+ _data = reinterpret_cast<pointer>(_stack);
+ }
+
+ std::memcpy(_data, rhs._data, _num * sizeof(T));
+ }
+
+ PassiveVector(PassiveVector&& rhs) : _num {rhs._num} {
+
+ // rhs is in the stack
+ if(rhs.in_stack()) {
+ _cap = S;
+ _data = reinterpret_cast<pointer>(_stack);
+ std::memcpy(_stack, rhs._stack, rhs._num*sizeof(T));
+ }
+ // rhs is in the heap
+ else {
+ _cap = rhs._cap;
+ _data = rhs._data;
+ rhs._data = reinterpret_cast<pointer>(rhs._stack);
+ rhs._cap = S;
+ }
+
+ rhs._num = 0;
+ }
+
+ ~PassiveVector() {
+ if(!in_stack()) {
+ _allocator.deallocate(_data, _cap);
+ }
+ }
+
+ iterator begin() noexcept { return _data; }
+ const_iterator begin() const noexcept { return _data; }
+ const_iterator cbegin() const noexcept { return _data; }
+ iterator end() noexcept { return _data + _num; }
+ const_iterator end() const noexcept { return _data + _num; }
+ const_iterator cend() const noexcept { return _data + _num; }
+
+ reverse_iterator rbegin() noexcept { return _data + _num; }
+ const_reverse_iterator crbegin() const noexcept { return _data + _num; }
+ reverse_iterator rend() noexcept { return _data; }
+ const_reverse_iterator crend() const noexcept { return _data; }
+
+ reference operator [] (size_type idx) { return _data[idx]; }
+ const_reference operator [] (size_type idx) const { return _data[idx]; }
+
+ reference at(size_type pos) {
+ if(pos >= _num) {
+ throw std::out_of_range("accessed position is out of range");
+ }
+ return this->operator[](pos);
+ }
+
+ const_reference at(size_type pos) const {
+ if(pos >= _num) {
+ throw std::out_of_range("accessed position is out of range");
+ }
+ return this->operator[](pos);
+ }
+
+
+ reference front() { return _data[0]; }
+ const_reference front() const { return _data[0]; }
+ reference back() { return _data[_num-1]; }
+ const_reference back() const { return _data[_num-1]; }
+
+ pointer data() noexcept { return _data; }
+ const_pointer data() const noexcept { return _data; }
+
+ void push_back(const T& item) {
+ if(_num == _cap) {
+ _enlarge(_cap << 1);
+ }
+ _data[_num++] = item;
+ }
+
+ void push_back(T&& item) {
+ if(_num == _cap) {
+ _enlarge(_cap << 1);
+ }
+ _data[_num++] = item;
+ }
+
+ void pop_back() {
+ if(_num > 0) {
+ --_num;
+ }
+ }
+
+ void clear() {
+ _num = 0;
+ }
+
+ void resize(size_type N) {
+ if(N > _cap) {
+ _enlarge(N<<1);
+ }
+ _num = N;
+ }
+
+ void reserve(size_type C) {
+ if(C > _cap) {
+ _enlarge(C);
+ }
+ }
+
+ bool empty() const { return _num == 0; }
+ bool in_stack() const { return _data == reinterpret_cast<const_pointer>(_stack); }
+
+ size_type size() const { return _num; }
+ size_type capacity() const { return _cap; }
+ size_type max_size() const { return std::numeric_limits<size_type>::max(); }
+
+ bool operator == (const PassiveVector& rhs) const {
+ if(_num != rhs._num) {
+ return false;
+ }
+ return std::memcmp(_data, rhs._data, _num * sizeof(T)) == 0;
+ }
+
+ private:
+
+ char _stack[S*sizeof(T)];
+
+ T* _data;
+
+ size_type _num;
+ size_type _cap;
+
+ A _allocator;
+
+ void _enlarge(size_type new_cap) {
+
+ auto new_data = _allocator.allocate(new_cap);
+
+ std::memcpy(new_data, _data, sizeof(T) * _num);
+
+ if(!in_stack()) {
+ _allocator.deallocate(_data, _cap);
+ }
+
+ _cap = new_cap;
+ _data = new_data;
+ }
+};
+
+
+} // end of namespace tf. ----------------------------------------------------
+
--- /dev/null
+#pragma once
+
+namespace tf {
+
+/** @class Singleton
+
+@brief class template to create a thread-safe singleton object
+
+*/
+template <typename T>
+class Singleton {
+
+ public:
+
+ /**
+ @brief get a reference to the singleton object
+ */
+ static T& get() {
+ static T instance;
+ return instance;
+ }
+
+ private:
+
+ Singleton() = default;
+ ~Singleton() = default;
+ Singleton(const Singleton&)= delete;
+ Singleton& operator=(const Singleton&)= delete;
+};
+
+
+
+} // end of namespace tf -----------------------------------------------------
--- /dev/null
+#pragma once
+
+#include <iostream>
+#include <string>
+
+namespace tf {
+
+// Procedure: stringify
+template <typename T>
+void ostreamize(std::ostringstream& oss, T&& token) {
+ oss << std::forward<T>(token);
+}
+
+// Procedure: stringify
+template <typename T, typename... Rest>
+void ostreamize(std::ostringstream& oss, T&& token, Rest&&... rest) {
+ oss << std::forward<T>(token);
+ ostreamize(oss, std::forward<Rest>(rest)...);
+}
+
+} // end of namespace tf -----------------------------------------------------
--- /dev/null
+#pragma once
+
+#include <type_traits>
+#include <iterator>
+#include <iostream>
+#include <fstream>
+#include <mutex>
+#include <deque>
+#include <vector>
+#include <algorithm>
+#include <memory>
+#include <atomic>
+#include <thread>
+#include <future>
+#include <functional>
+#include <unordered_map>
+#include <unordered_set>
+#include <sstream>
+#include <list>
+#include <forward_list>
+#include <numeric>
+#include <random>
+#include <iomanip>
+#include <cassert>
+#include <cmath>
+#include <cstring>
+
+#include "../nstd/variant.hpp"
+
+namespace tf {
+
+//-----------------------------------------------------------------------------
+// Traits
+//-----------------------------------------------------------------------------
+
+// Macro to check whether a class has a member function
+#define define_has_member(member_name) \
+template <typename T> \
+class has_member_##member_name \
+{ \
+ typedef char yes_type; \
+ typedef long no_type; \
+ template <typename U> static yes_type test(decltype(&U::member_name)); \
+ template <typename U> static no_type test(...); \
+ public: \
+ static constexpr bool value = sizeof(test<T>(0)) == sizeof(yes_type); \
+}
+
+#define has_member(class_, member_name) has_member_##member_name<class_>::value
+
+// Struct: dependent_false
+template <typename... T>
+struct dependent_false {
+ static constexpr bool value = false;
+};
+
+template <typename... T>
+constexpr auto dependent_false_v = dependent_false<T...>::value;
+
+//-----------------------------------------------------------------------------
+// Move-On-Copy
+//-----------------------------------------------------------------------------
+
+// Struct: MoC
+template <typename T>
+struct MoC {
+
+ MoC(T&& rhs) : object(std::move(rhs)) {}
+ MoC(const MoC& other) : object(std::move(other.object)) {}
+
+ T& get() { return object; }
+
+ mutable T object;
+};
+
+template <typename T>
+auto make_moc(T&& m) {
+ return MoC<T>(std::forward<T>(m));
+}
+
+//-----------------------------------------------------------------------------
+// Functors.
+//-----------------------------------------------------------------------------
+
+//// Overloadded.
+//template <typename... Ts>
+//struct Functors : Ts... {
+// using Ts::operator()... ;
+//};
+//
+//template <typename... Ts>
+//Functors(Ts...) -> Functors<Ts...>;
+
+// ----------------------------------------------------------------------------
+// callable traits
+// ----------------------------------------------------------------------------
+
+template <typename F, typename... Args>
+struct is_invocable :
+ std::is_constructible<
+ std::function<void(Args ...)>,
+ std::reference_wrapper<typename std::remove_reference<F>::type>
+ > {
+};
+
+template <typename F, typename... Args>
+constexpr bool is_invocable_v = is_invocable<F, Args...>::value;
+
+template <typename R, typename F, typename... Args>
+struct is_invocable_r :
+ std::is_constructible<
+ std::function<R(Args ...)>,
+ std::reference_wrapper<typename std::remove_reference<F>::type>
+ > {
+};
+
+template <typename R, typename F, typename... Args>
+constexpr bool is_invocable_r_v = is_invocable_r<R, F, Args...>::value;
+
+
+// ----------------------------------------------------------------------------
+// Function Traits
+// reference: https://github.com/ros2/rclcpp
+// ----------------------------------------------------------------------------
+
+template<typename T>
+struct tuple_tail;
+
+template<typename Head, typename ... Tail>
+struct tuple_tail<std::tuple<Head, Tail ...>> {
+ using type = std::tuple<Tail ...>;
+};
+
+// std::function
+template<typename F>
+struct function_traits
+{
+ using arguments = typename tuple_tail<
+ typename function_traits<decltype(&F::operator())>::argument_tuple_type
+ >::type;
+
+ static constexpr size_t arity = std::tuple_size<arguments>::value;
+
+ template <size_t N>
+ struct argument {
+ static_assert(N < arity, "error: invalid parameter index.");
+ using type = std::tuple_element_t<N, arguments>;
+ };
+
+ template <size_t N>
+ using argument_t = typename argument<N>::type;
+
+ using return_type = typename function_traits<decltype(&F::operator())>::return_type;
+};
+
+// Free functions
+template<typename R, typename... Args>
+struct function_traits<R(Args...)> {
+
+ using return_type = R;
+ using argument_tuple_type = std::tuple<Args...>;
+
+ static constexpr size_t arity = sizeof...(Args);
+
+ template <size_t N>
+ struct argument {
+ static_assert(N < arity, "error: invalid parameter index.");
+ using type = std::tuple_element_t<N, std::tuple<Args...>>;
+ };
+
+ template <size_t N>
+ using argument_t = typename argument<N>::type;
+};
+
+// function pointer
+template<typename R, typename... Args>
+struct function_traits<R(*)(Args...)> : function_traits<R(Args...)> {
+};
+
+// function reference
+template<typename R, typename... Args>
+struct function_traits<R(&)(Args...)> : function_traits<R(Args...)> {
+};
+
+// immutable lambda
+template<typename C, typename R, typename ... Args>
+struct function_traits<R(C::*)(Args ...) const>
+ : function_traits<R(C &, Args ...)>
+{};
+
+// mutable lambda
+template<typename C, typename R, typename ... Args>
+struct function_traits<R(C::*)(Args ...)>
+ : function_traits<R(C &, Args ...)>
+{};
+
+/*// std::bind for object methods
+template<typename C, typename R, typename ... Args, typename ... FArgs>
+#if defined _LIBCPP_VERSION // libc++ (Clang)
+struct function_traits<std::__bind<R (C::*)(Args ...), FArgs ...>>
+#elif defined _GLIBCXX_RELEASE // glibc++ (GNU C++ >= 7.1)
+struct function_traits<std::_Bind<R(C::*(FArgs ...))(Args ...)>>
+#elif defined __GLIBCXX__ // glibc++ (GNU C++)
+struct function_traits<std::_Bind<std::_Mem_fn<R (C::*)(Args ...)>(FArgs ...)>>
+#elif defined _MSC_VER // MS Visual Studio
+struct function_traits<
+ std::_Binder<std::_Unforced, R (C::*)(Args ...), FArgs ...>>
+#else
+#error "Unsupported C++ compiler / standard library"
+#endif
+ : function_traits<R(Args ...)>
+{};
+
+// std::bind for object const methods
+template<typename C, typename R, typename ... Args, typename ... FArgs>
+#if defined _LIBCPP_VERSION // libc++ (Clang)
+struct function_traits<std::__bind<R (C::*)(Args ...) const, FArgs ...>>
+#elif defined _GLIBCXX_RELEASE // glibc++ (GNU C++ >= 7.1)
+struct function_traits<std::_Bind<R(C::*(FArgs ...))(Args ...) const>>
+#elif defined __GLIBCXX__ // glibc++ (GNU C++)
+struct function_traits<std::_Bind<std::_Mem_fn<R (C::*)(Args ...) const>(FArgs ...)>>
+#elif defined _MSC_VER // MS Visual Studio
+struct function_traits<
+ std::_Binder<std::_Unforced, R (C::*)(Args ...) const, FArgs ...>>
+#else
+#error "Unsupported C++ compiler / standard library"
+#endif
+ : function_traits<R(Args ...)>
+{};
+
+// std::bind for free functions
+template<typename R, typename ... Args, typename ... FArgs>
+#if defined _LIBCPP_VERSION // libc++ (Clang)
+struct function_traits<std::__bind<R( &)(Args ...), FArgs ...>>
+#elif defined __GLIBCXX__ // glibc++ (GNU C++)
+struct function_traits<std::_Bind<R(*(FArgs ...))(Args ...)>>
+#elif defined _MSC_VER // MS Visual Studio
+struct function_traits<std::_Binder<std::_Unforced, R( &)(Args ...), FArgs ...>>
+#else
+#error "Unsupported C++ compiler / standard library"
+#endif
+ : function_traits<R(Args ...)>
+{}; */
+
+// decay to the raw type
+template <typename F>
+struct function_traits<F&> : function_traits<F> {};
+
+template <typename F>
+struct function_traits<F&&> : function_traits<F> {};
+
+
+// ----------------------------------------------------------------------------
+// nstd::variant
+// ----------------------------------------------------------------------------
+template <typename T, typename>
+struct get_index;
+
+template <size_t I, typename... Ts>
+struct get_index_impl {};
+
+template <size_t I, typename T, typename... Ts>
+struct get_index_impl<I, T, T, Ts...> : std::integral_constant<size_t, I>{};
+
+template <size_t I, typename T, typename U, typename... Ts>
+struct get_index_impl<I, T, U, Ts...> : get_index_impl<I+1, T, Ts...>{};
+
+template <typename T, typename... Ts>
+struct get_index<T, nstd::variant<Ts...>> : get_index_impl<0, T, Ts...>{};
+
+template <typename T, typename... Ts>
+constexpr auto get_index_v = get_index<T, Ts...>::value;
+
+// ----------------------------------------------------------------------------
+// is_pod
+//-----------------------------------------------------------------------------
+template <typename T>
+struct is_pod {
+ static const bool value = std::is_trivial<T>::value &&
+ std::is_standard_layout<T>::value;
+};
+
+template <typename T>
+constexpr bool is_pod_v = is_pod<T>::value;
+
+// ----------------------------------------------------------------------------
+// bit_cast
+//-----------------------------------------------------------------------------
+template <class To, class From>
+typename std::enable_if<
+ (sizeof(To) == sizeof(From)) &&
+ std::is_trivially_copyable<From>::value &&
+ std::is_trivial<To>::value,
+ // this implementation requires that To is trivially default constructible
+ To
+>::type
+// constexpr support needs compiler magic
+bit_cast(const From &src) noexcept {
+ To dst;
+ std::memcpy(&dst, &src, sizeof(To));
+ return dst;
+}
+
+} // end of namespace tf. ---------------------------------------------------
+
+
+
--- /dev/null
+#pragma once
+
+#include <iostream>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cstddef>
+#include <random>
+#include <chrono>
+
+namespace tf {
+
+// Class: UUID
+//
+// A universally unique identifier (UUID) is an identifier standard used in software
+// construction. A UUID is simply a 128-bit value. The meaning of each bit is defined
+// by any of several variants.
+// For human-readable display, many systems use a canonical format using hexadecimal
+// text with inserted hyphen characters.
+//
+// For example: 123e4567-e89b-12d3-a456-426655440000
+//
+// The intent of UUIDs is to enable distributed systems to uniquely identify information
+// without significant central coordination.
+//
+// Copyright 2006 Andy Tompkins.
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+//
+struct UUID {
+
+ using value_type = uint8_t;
+ using reference = uint8_t&;
+ using const_reference = const uint8_t&;
+ using iterator = uint8_t*;
+ using const_iterator = const uint8_t*;
+ using size_type = size_t;
+ using difference_type = ptrdiff_t;
+
+ inline UUID();
+
+ UUID(const UUID&) = default;
+ UUID(UUID&&) = default;
+
+ UUID& operator = (const UUID&) = default;
+ UUID& operator = (UUID&&) = default;
+
+ inline static size_type size();
+ inline iterator begin();
+ inline const_iterator begin() const;
+ inline iterator end();
+ inline const_iterator end() const;
+
+ inline bool is_nil() const;
+ inline void swap(UUID& rhs);
+ inline size_t hash_value() const;
+
+ inline bool operator == (const UUID&) const;
+ inline bool operator < (const UUID&) const;
+ inline bool operator > (const UUID&) const;
+ inline bool operator != (const UUID&) const;
+ inline bool operator >= (const UUID&) const;
+ inline bool operator <= (const UUID&) const;
+
+ uint8_t data[16] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+ inline std::string to_string() const;
+};
+
+// Constructor
+inline UUID::UUID() {
+
+ static thread_local std::random_device rd;
+ static thread_local std::mt19937 engine {rd()};
+
+ std::uniform_int_distribution<unsigned long> distribution(
+ std::numeric_limits<unsigned long>::min(),
+ std::numeric_limits<unsigned long>::max()
+ );
+
+ int i = 0;
+ auto random_value = distribution(engine);
+ for (auto it=begin(); it!=end(); ++it, ++i) {
+ if (i == sizeof(unsigned long)) {
+ random_value = distribution(engine);
+ i = 0;
+ }
+ *it = static_cast<UUID::value_type>((random_value >> (i*8)) & 0xFF);
+ }
+
+ // set variant: must be 0b10xxxxxx
+ *(begin()+8) &= 0xBF;
+ *(begin()+8) |= 0x80;
+
+ // set version: must be 0b0100xxxx
+ *(begin()+6) &= 0x4F; //0b01001111
+ *(begin()+6) |= 0x40; //0b01000000
+}
+
+// Function: size
+inline typename UUID::size_type UUID::size() {
+ return 16;
+}
+
+// Function: begin
+inline typename UUID::iterator UUID::begin() {
+ return data;
+}
+
+// Function: begin
+inline typename UUID::const_iterator UUID::begin() const {
+ return data;
+}
+
+// Function: end
+inline typename UUID::iterator UUID::end() {
+ return data+size();
+}
+
+// Function: end
+inline typename UUID::const_iterator UUID::end() const {
+ return data+size();
+}
+
+// Function: is_nil
+inline bool UUID::is_nil() const {
+ for (std::size_t i = 0; i < sizeof(this->data); ++i) {
+ if (this->data[i] != 0U) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Procedure: swap
+inline void UUID::swap(UUID& rhs) {
+ UUID tmp = *this;
+ *this = rhs;
+ rhs = tmp;
+}
+
+// Function: hash_value
+inline size_t UUID::hash_value() const {
+ size_t seed = 0;
+ for(auto i=begin(); i != end(); ++i) {
+ seed ^= static_cast<size_t>(*i) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+ }
+ return seed;
+}
+
+// Operator: ==
+inline bool UUID::operator == (const UUID& rhs) const {
+ return std::memcmp(data, rhs.data, sizeof(data)) == 0;
+}
+
+// Operator: !=
+inline bool UUID::operator != (const UUID& rhs) const {
+ return std::memcmp(data, rhs.data, sizeof(data)) != 0;
+}
+
+// Operator: <
+inline bool UUID::operator < (const UUID& rhs) const {
+ return std::memcmp(data, rhs.data, sizeof(data)) < 0;
+}
+
+// Operator: >
+inline bool UUID::operator > (const UUID& rhs) const {
+ return std::memcmp(data, rhs.data, sizeof(data)) > 0;
+}
+
+// Operator: <=
+inline bool UUID::operator <= (const UUID& rhs) const {
+ return std::memcmp(data, rhs.data, sizeof(data)) <= 0;
+}
+
+// Operator: >=
+inline bool UUID::operator >= (const UUID& rhs) const {
+ return std::memcmp(data, rhs.data, sizeof(data)) >= 0;
+}
+
+// Function: to_string
+inline std::string UUID::to_string() const {
+
+ auto to_char = [](size_t i) {
+ if (i <= 9) return static_cast<char>('0' + i);
+ return static_cast<char>('a' + (i-10));
+ };
+
+ std::string result;
+ result.reserve(36);
+
+ std::size_t i=0;
+ for (auto it = begin(); it!=end(); ++it, ++i) {
+
+ const size_t hi = ((*it) >> 4) & 0x0F;
+ result += to_char(hi);
+
+ const size_t lo = (*it) & 0x0F;
+ result += to_char(lo);
+
+ if (i == 3 || i == 5 || i == 7 || i == 9) {
+ result += '-';
+ }
+ }
+ return result;
+}
+
+// Procedure: swap
+inline void swap(UUID& lhs, UUID& rhs) {
+ lhs.swap(rhs);
+}
+
+// ostream
+inline std::ostream& operator << (std::ostream& os, const UUID& rhs) {
+ os << rhs.to_string();
+ return os;
+}
+
+}; // End of namespace tf. ---------------------------------------------------
+
+//-----------------------------------------------------------------------------
+
+
+namespace std {
+
+// Partial specialization: hash<tf::UUID>
+template <>
+struct hash<tf::UUID> {
+ size_t operator()(const tf::UUID& rhs) const { return rhs.hash_value(); }
+};
+
+
+} // End of namespace std. ---------------------------------------------------
+
+
SET(BOOST_FOLDER "${CMAKE_SOURCE_DIR}/bundled/boost-1.70.0")
+#
+# CPP Taskflow
+#
+
+SET(FEATURE_TASKFLOW_HAVE_BUNDLED TRUE)
+
+OPTION(DEAL_II_FORCE_BUNDLED_TASKFLOW
+ "Always use the bundled taskflow header library instead of an external one."
+ OFF)
+
+SET(TASKFLOW_FOLDER "${CMAKE_SOURCE_DIR}/bundled/cpp-taskflow")
+
#
# Threading Building Blocks library
#
# Cygwin is unsupported by tbb, Windows due to the way we compile tbb...
#
- SET(FEATURE_THREADS_HAVE_BUNDLED TRUE)
+ SET(FEATURE_TBB_HAVE_BUNDLED TRUE)
- OPTION(DEAL_II_FORCE_BUNDLED_THREADS
+ OPTION(DEAL_II_FORCE_BUNDLED_TBB
"Always use the bundled tbb library instead of an external one."
OFF)
--- /dev/null
+## ---------------------------------------------------------------------
+##
+## Copyright (C) 2012 - 2020 by the deal.II authors
+##
+## This file is part of the deal.II library.
+##
+## The deal.II library is free software; you can use it, redistribute
+## it, and/or modify it under the terms of the GNU Lesser General
+## Public License as published by the Free Software Foundation; either
+## version 2.1 of the License, or (at your option) any later version.
+## The full text of the license can be found in the file LICENSE.md at
+## the top level directory of deal.II.
+##
+## ---------------------------------------------------------------------
+
+#
+# Configuration for thread support in deal.II with the help of the CPP
+# Taskflow library:
+#
+
+
+MACRO(FEATURE_CPP_TASKFLOW_CONFIGURE_BUNDLED)
+ LIST(APPEND CPP_TASKFLOW_BUNDLED_INCLUDE_DIRS ${CPP_TASKFLOW_FOLDER}/include)
+ENDMACRO()
+
+
+CONFIGURE_FEATURE(CPP_TASKFLOW)
+
ENDMACRO()
-MACRO(FEATURE_TBB_CONFIGURE_EXTERNAL)
-ENDMACRO()
-
-
MACRO(FEATURE_TBB_CONFIGURE_BUNDLED)
#
# We have to disable a bunch of warnings:
--- /dev/null
+## ---------------------------------------------------------------------
+##
+## Copyright (C) 2020 by the deal.II authors
+##
+## This file is part of the deal.II library.
+##
+## The deal.II library is free software; you can use it, redistribute
+## it, and/or modify it under the terms of the GNU Lesser General
+## Public License as published by the Free Software Foundation; either
+## version 2.1 of the License, or (at your option) any later version.
+## The full text of the license can be found in the file LICENSE.md at
+## the top level directory of deal.II.
+##
+## ---------------------------------------------------------------------
+
+#
+# Try to find the Threading Building Blocks library
+#
+# This module exports
+#
+# CPP_TASKFLOW_INCLUDE_DIRS
+# CPP_TASKFLOW_VERSION
+#
+
+SET(CPP_TASKFLOW_DIR "" CACHE PATH "An optional hint to a CPP CPP_TASKFLOW installation")
+SET_IF_EMPTY(CPP_TASKFLOW_DIR "$ENV{CPP_TASKFLOW_DIR}")
+
+FIND_PACKAGE(CPP_TASKFLOW_CONFIG
+ CONFIG QUIET
+ NAMES Cpp-Taskflow
+ HINTS
+ ${CPP_TASKFLOW_DIR}/lib/cmake/Cpp-Taskflow
+ ${CPP_TASKFLOW_DIR}
+ PATH_SUFFIXES
+ lib64/cmake/Cpp-Taskflow
+ lib/cmake/Cpp-Taskflow
+ lib${LIB_SUFFIX}/cmake/Cpp-Taskflow
+ NO_SYSTEM_ENVIRONMENT_PATH
+ )
+
+SET(CPP_TASKFLOW_INCLUDE_DIR ${Cpp-Taskflow_INCLUDE_DIR})
+
+#
+# Extract version numbers:
+#
+SET(CPP_TASKFLOW_VERSION "${CPP_TASKFLOW_CONFIG_VERSION}")
+STRING(REGEX REPLACE
+ "^([0-9]+).*$" "\\1"
+ CPP_TASKFLOW_VERSION_MAJOR "${CPP_TASKFLOW_CONFIG_VERSION}")
+STRING(REGEX REPLACE
+ "^[0-9]+\\.([0-9]+).*$" "\\1"
+ CPP_TASKFLOW_VERSION_MINOR "${CPP_TASKFLOW_CONFIG_VERSION}")
+
+DEAL_II_PACKAGE_HANDLE(CPP_TASKFLOW
+ INCLUDE_DIRS REQUIRED CPP_TASKFLOW_INCLUDE_DIR
+ USER_INCLUDE_DIRS REQUIRED CPP_TASKFLOW_INCLUDE_DIR
+ CLEAR CPP_TASKFLOW_CONFIG_DIR
+ )
projects / dealii.git / commitdiff