Restructure data of parallel regions and implicit tasks.

jblueh · jblueh · commit a1e212e561c0 · 2024-08-19T19:10:36.000+02:00
Associate tapes, positions, and adjoint access modes with tasks instead of parallel regions.
Extend data of parallel regions by pointers to the parent task and child tasks.

Build tests with output instrument.

Merge branch 'feature/restructureTaskData' into develop
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -39,3 +39,15 @@ test-clang-ompt:
     - export BACKEND=OMPT
     - make all
 
+test-output:
+  image: ubuntu:22.04
+  timeout: 2h
+  script:
+    - apt update && apt install -y build-essential binutils git
+    - git clone --depth 1 --branch develop https://github.com/SciCompKL/CoDiPack.git
+    - export CODI_DIR=$(pwd)/CoDiPack/include
+    - export OPDI_DIR=$(pwd)/include
+    - cd tests
+    - export CXX=g++
+    - export OUTPUT_INSTRUMENT=yes
+    - make all
diff --git a/include/opdi/backend/macro/dataTools.hpp b/include/opdi/backend/macro/dataTools.hpp
@@ -34,6 +34,9 @@ namespace opdi {
       static std::stack<void*> parallelData;
       #pragma omp threadprivate(parallelData)
 
+      static std::stack<void*> taskData;
+      #pragma omp threadprivate(taskData)
+
     public:
       static void pushParallelData(void* parallelData) {
         DataTools::parallelData.push(parallelData);
@@ -50,5 +53,21 @@ namespace opdi {
 
         return DataTools::parallelData.top();
       }
+
+      static void pushTaskData(void* taskData) {
+        DataTools::taskData.push(taskData);
+      }
+
+      static void popTaskData() {
+        DataTools::taskData.pop();
+      }
+
+      static void* getTaskData() {
+        if (DataTools::taskData.empty()) {
+          return nullptr;
+        }
+
+        return DataTools::taskData.top();
+      }
   };
 }
diff --git a/include/opdi/backend/macro/macroBackend.cpp b/include/opdi/backend/macro/macroBackend.cpp
@@ -30,6 +30,7 @@
 // static macro backend members
 
 std::stack<void*> opdi::DataTools::parallelData;
+std::stack<void*> opdi::DataTools::taskData;
 
 std::stack<bool> opdi::ImplicitBarrierTools::implicitBarrierStack;
 
diff --git a/include/opdi/backend/macro/macros.hpp b/include/opdi/backend/macro/macros.hpp
@@ -38,7 +38,7 @@
 
 #define OPDI_PARALLEL(...) \
   { \
-    void* opdiInternalParallelData = opdi::logic->onParallelBegin(omp_get_max_threads()); \
+    void* opdiInternalParallelData = opdi::logic->onParallelBegin(opdi::DataTools::getTaskData(), omp_get_max_threads()); \
     opdi::TaskProbe opdiInternalTaskProbe(opdiInternalParallelData); \
     OPDI_PRAGMA(omp parallel __VA_ARGS__ firstprivate(opdiInternalTaskProbe))
 
diff --git a/include/opdi/backend/macro/probes.hpp b/include/opdi/backend/macro/probes.hpp
@@ -71,6 +71,7 @@ namespace opdi {
 
         DataTools::pushParallelData(this->parallelData);
         this->taskData = logic->onImplicitTaskBegin(omp_get_num_threads(), omp_get_thread_num(), this->parallelData);
+        DataTools::pushTaskData(this->taskData);
 
         // check if copy statements have been recorded before the correct tape was set
         // if so, move them to the correct tape
@@ -88,6 +89,7 @@ namespace opdi {
         if (needsAction) {
           ProbeScopeStatus::endImplicitTaskProbeScope();
           logic->onImplicitTaskEnd(this->taskData);
+          DataTools::popTaskData();
           DataTools::popParallelData();
         }
 
diff --git a/include/opdi/backend/ompt/parallelCallbacks.hpp b/include/opdi/backend/ompt/parallelCallbacks.hpp
@@ -52,13 +52,12 @@ namespace opdi {
           int flags,
           void const* codeptr) {
 
-        OPDI_UNUSED(encounteringTaskData);
         OPDI_UNUSED(encounteringTaskFrame);
         OPDI_UNUSED(parallelData);
         OPDI_UNUSED(flags);
         OPDI_UNUSED(codeptr);
 
-        parallelData->ptr = logic->onParallelBegin(requestedParallelism);
+        parallelData->ptr = logic->onParallelBegin(encounteringTaskData->ptr, requestedParallelism);
       }
 
       static void onParallelEnd(
diff --git a/include/opdi/logic/logicInterface.hpp b/include/opdi/logic/logicInterface.hpp
@@ -55,7 +55,7 @@ namespace opdi {
 
       virtual ~LogicInterface() {}
 
-      virtual void* onParallelBegin(int maxThreads) = 0;
+      virtual void* onParallelBegin(void* encounteringTaskData, int maxThreads) = 0;
       virtual void onParallelEnd(void* data) = 0;
 
       virtual void* onImplicitTaskBegin(int actualParallelism, int index, void* parallelData) = 0;
diff --git a/include/opdi/logic/omp/implicitTaskOmpLogic.cpp b/include/opdi/logic/omp/implicitTaskOmpLogic.cpp
@@ -55,28 +55,30 @@ void* opdi::ImplicitTaskOmpLogic::onImplicitTaskBegin(int actualParallelism, int
     data->oldTape = tool->getThreadLocalTape();
     data->parallelData = parallelData;
 
-    void* newTape = this->tapePool.getTape(parallelData->masterTape, index);
+    void* newTape = this->tapePool.getTape(parallelData->parentTape, index);
 
     if (parallelData->activeParallelRegion) {
       tool->setActive(newTape, true);
     }
 
-    data->parallelData->tapes[index] = newTape;
+    data->tape = newTape;
 
-    data->parallelData->positions[index].push_back(tool->allocPosition());
-    tool->getTapePosition(newTape, data->parallelData->positions[index].back());
+    data->positions.push_back(tool->allocPosition());
+    tool->getTapePosition(newTape, data->positions.back());
 
     tool->setThreadLocalTape(newTape);
 
-    AdjointAccessControl::pushMode(data->parallelData->outerAdjointAccessMode);
-    data->parallelData->adjointAccessModes[index].push_back(data->parallelData->outerAdjointAccessMode);
+    AdjointAccessControl::pushMode(parallelData->parentAdjointAccessMode);
+    data->adjointAccessModes.push_back(parallelData->parentAdjointAccessMode);
 
     #if OPDI_OMP_LOGIC_INSTRUMENT
       for (auto& instrument : ompLogicInstruments) {
         instrument->onImplicitTaskBegin(data);
       }
     #endif
 
+    parallelData->childTasks[index] = data;
+
     return data;
   }
 
@@ -94,13 +96,11 @@ void opdi::ImplicitTaskOmpLogic::onImplicitTaskEnd(void* dataPtr) {
 
     tool->setThreadLocalTape(data->oldTape);
 
-    data->parallelData->positions[data->index].push_back(tool->allocPosition());
-    tool->getTapePosition(data->parallelData->tapes[data->index],
-                          data->parallelData->positions[data->index].back());
+    data->positions.push_back(tool->allocPosition());
+    tool->getTapePosition(data->tape, data->positions.back());
 
     if (!data->parallelData->activeParallelRegion) {
-      if (tool->comparePosition(data->parallelData->positions[data->index].front(),
-                                data->parallelData->positions[data->index].back()) != 0) {
+      if (tool->comparePosition(data->positions.front(), data->positions.back()) != 0) {
         OPDI_WARNING("Something became active during a passive parallel region. This is not supported and will not be ",
                      "differentiated correctly.");
       }
@@ -112,12 +112,13 @@ void opdi::ImplicitTaskOmpLogic::onImplicitTaskEnd(void* dataPtr) {
       }
     #endif
 
-    tool->setActive(data->parallelData->tapes[data->index], false);
+    tool->setActive(data->tape, false);
 
-    if (data->oldTape == data->parallelData->masterTape && data->parallelData->activeParallelRegion) {
+    // ensure that the most recent activity change *per thread* reflects the current activity
+    if (data->oldTape == data->parallelData->parentTape && data->parallelData->activeParallelRegion) {
       tool->setActive(data->oldTape, true);
     }
 
-    delete data;
+    // do not delete data, it is deleted as part of parallel regions
   }
 }
diff --git a/include/opdi/logic/omp/implicitTaskOmpLogic.hpp b/include/opdi/logic/omp/implicitTaskOmpLogic.hpp
@@ -25,6 +25,8 @@
 
 #pragma once
 
+#include <deque>
+
 #include "../../misc/tapePool.hpp"
 
 #include "../logicInterface.hpp"
@@ -44,14 +46,19 @@ namespace opdi {
 
     public:
 
+      using LogicInterface::AdjointAccessMode;
+
       using ParallelData = typename ParallelOmpLogic::Data;
 
       struct Data {
         public:
           int level;
           int index;
           void* oldTape;
+          void* tape;
           ParallelData* parallelData;
+          std::deque<void*> positions;
+          std::deque<AdjointAccessMode> adjointAccessModes;
       };
 
       virtual void* onImplicitTaskBegin(int actualParallelism, int index, void* parallelDataPtr);
diff --git a/include/opdi/logic/omp/instrument/ompLogicInstrumentInterface.hpp b/include/opdi/logic/omp/instrument/ompLogicInstrumentInterface.hpp
@@ -45,9 +45,9 @@ namespace opdi {
 
       virtual void reverseFlush() {}
 
-      virtual void reverseImplicitTaskBegin(ParallelOmpLogic::Data* /*data*/, int /*threadNum*/) {}
-      virtual void reverseImplicitTaskEnd(ParallelOmpLogic::Data* /*data*/, int /*threadNum*/) {}
-      virtual void reverseImplicitTaskPart(ParallelOmpLogic::Data* /*data*/, int /*threadNum*/, std::size_t /*part*/) {}
+      virtual void reverseImplicitTaskBegin(ImplicitTaskOmpLogic::Data* /*data*/) {}
+      virtual void reverseImplicitTaskEnd(ImplicitTaskOmpLogic::Data* /*data*/) {}
+      virtual void reverseImplicitTaskPart(ImplicitTaskOmpLogic::Data* /*data*/, std::size_t /*part*/) {}
       virtual void onImplicitTaskBegin(ImplicitTaskOmpLogic::Data* /*data*/) {}
       virtual void onImplicitTaskEnd(ImplicitTaskOmpLogic::Data* /*data*/) {}
 
diff --git a/include/opdi/logic/omp/instrument/ompLogicOutputInstrument.hpp b/include/opdi/logic/omp/instrument/ompLogicOutputInstrument.hpp
@@ -40,41 +40,41 @@ namespace opdi {
         TapedOutput::print("R FLSH l", omp_get_level(), "t", omp_get_thread_num());
       }
 
-      virtual void reverseImplicitTaskBegin(ParallelOmpLogic::Data* data, int threadNum) {
+      virtual void reverseImplicitTaskBegin(ImplicitTaskOmpLogic::Data* data) {
         TapedOutput::print("R IMTB l", omp_get_level(),
-                           "t", threadNum,
-                           "tape", data->tapes[threadNum],
-                           "pos", tool->positionToString(data->positions[threadNum].back()));
+                           "t", data->index,
+                           "tape", data->tape,
+                           "pos", tool->positionToString(data->positions.back()));
       }
 
-      virtual void reverseImplicitTaskEnd(ParallelOmpLogic::Data* data, int threadNum) {
+      virtual void reverseImplicitTaskEnd(ImplicitTaskOmpLogic::Data* data) {
         TapedOutput::print("R IMTE l", omp_get_level(),
-                           "t", threadNum,
-                           "tape", data->tapes[threadNum],
-                           "pos", tool->positionToString(data->positions[threadNum].front()));
+                           "t", data->index,
+                           "tape", data->tape,
+                           "pos", tool->positionToString(data->positions.front()));
       }
 
-      virtual void reverseImplicitTaskPart(ParallelOmpLogic::Data* data, int threadNum, std::size_t part) {
+      virtual void reverseImplicitTaskPart(ImplicitTaskOmpLogic::Data* data, std::size_t part) {
         TapedOutput::print("R IMTP l", omp_get_level(),
-                           "t", threadNum,
-                           "tape", data->tapes[threadNum],
-                           "start", tool->positionToString(data->positions[threadNum][part]),
-                           "end", tool->positionToString(data->positions[threadNum][part - 1]),
-                           "mode", data->adjointAccessModes[threadNum][part - 1]);
+                           "t", data->index,
+                           "tape", data->tape,
+                           "start", tool->positionToString(data->positions[part]),
+                           "end", tool->positionToString(data->positions[part - 1]),
+                           "mode", data->adjointAccessModes[part - 1]);
       }
 
       virtual void onImplicitTaskBegin(ImplicitTaskOmpLogic::Data* data) {
         TapedOutput::print("F IMTB l", omp_get_level(),
                            "t", data->index,
-                           "tape", data->parallelData->tapes[data->index],
-                           "pos", tool->positionToString(data->parallelData->positions[data->index].back()));
+                           "tape", data->tape,
+                           "pos", tool->positionToString(data->positions.back()));
       }
 
       virtual void onImplicitTaskEnd(ImplicitTaskOmpLogic::Data* data) {
         TapedOutput::print("F IMTE l", omp_get_level(),
                            "t", data->index,
-                           "tape", data->parallelData->tapes[data->index],
-                           "pos", tool->positionToString(data->parallelData->positions[data->index].back()));
+                           "tape", data->tape,
+                           "pos", tool->positionToString(data->positions.back()));
       }
 
       virtual void reverseMutexWait(MutexOmpLogic::Data* data) {
@@ -114,12 +114,12 @@ namespace opdi {
 
       virtual void reverseParallelBegin(ParallelOmpLogic::Data* data) {
         TapedOutput::print("R PARB l", omp_get_level(),
-                           "master", data->masterTape);
+                           "parent", data->parentTape);
       }
 
       virtual void reverseParallelEnd(ParallelOmpLogic::Data* data) {
         TapedOutput::print("R PARE l", omp_get_level(),
-                           "master", data->masterTape);
+                           "parent", data->parentTape);
       }
 
       virtual void onParallelBegin(ParallelOmpLogic::Data* data) {
@@ -129,8 +129,8 @@ namespace opdi {
         }
         else {
           TapedOutput::print("F PARB l", omp_get_level(),
-                             "master", data->masterTape,
-                             "mode", data->outerAdjointAccessMode);
+                             "parent", data->parentTape,
+                             "mode", data->parentAdjointAccessMode);
         }
       }
 
@@ -141,8 +141,8 @@ namespace opdi {
         }
         else {
           TapedOutput::print("F PARE l", omp_get_level(),
-                             "master", data->masterTape,
-                             "mode", data->outerAdjointAccessMode);
+                             "parent", data->parentTape,
+                             "mode", data->parentAdjointAccessMode);
         }
       }
 
diff --git a/include/opdi/logic/omp/parallelOmpLogic.cpp b/include/opdi/logic/omp/parallelOmpLogic.cpp
diff --git a/include/opdi/logic/omp/parallelOmpLogic.hpp b/include/opdi/logic/omp/parallelOmpLogic.hpp
diff --git a/tests/Makefile b/tests/Makefile
diff --git a/tests/run.sh b/tests/run.sh
