add remap_id to pintool

Author: plavin

src/sst/elements/ariel/api/arielapi.c

@@ -17,9 +17,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <inttypes.h>
+/*
 #ifdef ENABLE_ARIEL_MPI
 #include <mpi.h>
 #endif
+*/
 
 /* These definitions are replaced during simulation */
 
@@ -36,6 +38,7 @@ void ariel_fence() {
 }
 
 uint64_t ariel_cycles() {
+    printf("ARIEL: ariel_cycles called in Ariel API.\n");
     return 0;
 }
 
@@ -47,6 +50,7 @@ void ariel_malloc_flag(int64_t id, int count, int level) {
     printf("ARIEL: flagging next %d mallocs at id %" PRId64 "\n", count, id);
 }
 
+/*
 // To ensure that the Pintool (fesimple.cc) numbers our application's OpenMP threads
 // from 0..N-1, we need to run an OpenMP parallel region before calling MPI Init.
 // Otherwise, some MPI threads which aren't used for our application will be
@@ -103,3 +107,4 @@ int MPI_Init_thread(int *argc, char ***argv, int required, int *provided) {
     exit(1);
 #endif
 }
+*/

src/sst/elements/ariel/frontend/pin3/fesimple.cc

@@ -22,6 +22,9 @@
 #include "atomic.hpp"
 #include <time.h>
 #include <inttypes.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <unordered_map>
 
 #include <string>
 #include <map>
@@ -180,6 +183,96 @@ class StackRecord {
 
 std::vector<std::vector<StackRecord> > arielStack; // Per-thread stacks
 
+static const UINT32 MAX_FRAMES = 64;
+bool is_mpi_thread(CONTEXT* ctxt) {
+    void *buf[256];
+
+    // PIN_Backtrace fills the trace array with return addresses.
+    // numFrames holds the number of captured frames.
+    UINT32 numFrames = PIN_Backtrace(ctxt, buf, sizeof(buf) / sizeof(buf[0]));
+    //std::cout << "numFrames: " << numFrames << std::endl;
+
+    for (UINT32 i = 0; i < numFrames; i++)
+    {
+        // Get the image (module) that contains the current address.
+        IMG img = IMG_FindByAddress((ADDRINT)buf[i]);
+        if (IMG_Valid(img))
+        {
+            // Retrieve the image name as a std::string.
+            std::string imgName = IMG_Name(img);
+            //std::cout << "imgName: " << imgName << std::endl;
+            // Check if "libmpi.so" appears in the module name.
+            if (imgName.find("libmpi.so") != std::string::npos)
+            {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+UINT32 num_threads;
+
+// Used for synchronizing access to the thread id map
+std::unordered_map<THREADID, THREADID> remap_id;
+
+TLS_KEY syscall_id;
+TLS_KEY clone_is_mpi;
+TLS_KEY _os_tid;
+VOID ThreadStart(THREADID tid, CONTEXT *ctxt, INT32 flags, VOID* v) {
+   // Initialize data for storing syscall number as it is only availble on Entry, not Exit
+   PIN_SetThreadData(syscall_id, new ADDRINT(0), tid);
+   PIN_SetThreadData(clone_is_mpi, new bool(false), tid);
+   
+   //PIN_SetThreadData(_os_tid, new ADDRINT(0), tid);
+}
+/*
+VOID ThreadFini(THREADID tid, CONTEXT *ctxt, INT32 flags, VOID* v) {
+   // Delete thread local storage
+   ADDRINT *syscallNum = static_cast<ADDRINT *>(PIN_GetThreadData(tlsKey, threadId));
+   delete syscallNum;
+}
+*/
+UINT32 next_app_thread;
+UINT32 next_mpi_thread;
+
+VOID SyscallEntry(THREADID threadid, CONTEXT *ctxt, SYSCALL_STANDARD std, VOID *v)
+{
+   // Retrieve the system call number.
+   PIN_LockClient();
+   ADDRINT scNo = PIN_GetSyscallNumber(ctxt, std);
+   ADDRINT *syscall_id_ptr = static_cast<ADDRINT *>(PIN_GetThreadData(syscall_id, threadid));
+   *syscall_id_ptr = scNo;
+
+
+   bool is_mpi = false;
+   if (scNo == SYS_clone || scNo == __NR_clone)
+   {
+       PIN_GetLock(&mainLock, threadid);
+       UINT32 next_thread = num_threads;
+       num_threads++;
+
+       bool is_mpi = is_mpi_thread(ctxt);
+       std::cout << "next_thread is " << next_thread << std::endl;
+       if (is_mpi) {
+          remap_id[next_thread] = next_mpi_thread++;
+       } else {
+          remap_id[next_thread] = next_app_thread++;
+       }
+       PIN_ReleaseLock(&mainLock);
+
+       std::cout << "--------------\n";
+       for (const auto& pair : remap_id) {
+         std::cout << " --> " << pair.first << " -- " << pair.second << std::endl;
+       }
+       std::cout << "--------------\n";
+
+   }
+   bool *clone_is_mpi_ptr = static_cast<bool *>(PIN_GetThreadData(clone_is_mpi, threadid));
+   *clone_is_mpi_ptr = is_mpi;
+   PIN_UnlockClient();
+}
+
 /* Instrumentation function to be called on function calls */
 VOID ariel_stack_call(THREADID thr, ADDRINT stackPtr, ADDRINT target, ADDRINT ip)
 {
@@ -833,17 +926,21 @@ void mapped_ariel_fence(void *virtualAddress)
     WriteFenceInstructionMarker(thr, ip);
 }
 
+/*
 void mapped_api_mpi_init() {
     api_mpi_init_used = 1;
 }
+*/
 
+/*
 int check_for_api_mpi_init() {
     if (!api_mpi_init_used && !getenv("ARIEL_DISABLE_MPI_INIT_CHECK")) {
         fprintf(stderr, "Error: fesimple.cc: The Ariel API verion of MPI_Init_{thread} was not used, which can result in errors when used in conjunction with OpenMP. Please link against the Ariel API (included in this distribution at src/sst/elements/ariel/api) or disable this message by setting the environment variable `ARIEL_DISABLE_MPI_INIT_CHECK`\n");
         exit(1);
     }
     return 0;
 }
+*/
 
 int ariel_mlm_memcpy(void* dest, void* source, size_t size) {
 #ifdef ARIEL_DEBUG
@@ -1240,6 +1337,7 @@ VOID InstrumentRoutine(RTN rtn, VOID* args)
         RTN_Replace(rtn, (AFUNPTR) mapped_ariel_cycles);
         fprintf(stderr, "Replacement complete\n");
         return;
+        /*
     } else if (RTN_Name(rtn) == "MPI_Init" || RTN_Name(rtn) == "_MPI_Init") {
         fprintf(stderr, "Identified routine: MPI_Init. Instrumenting.\n");
         RTN_Open(rtn);
@@ -1257,7 +1355,7 @@ VOID InstrumentRoutine(RTN rtn, VOID* args)
         RTN_Replace(rtn, (AFUNPTR) mapped_api_mpi_init);
         fprintf(stderr, "Replacement complete\n");
         return;
-        return;
+        */
 #if ! defined(__APPLE__)
     } else if (RTN_Name(rtn) == "clock_gettime" || RTN_Name(rtn) == "_clock_gettime" ||
         RTN_Name(rtn) == "__clock_gettime") {
@@ -1276,7 +1374,6 @@ VOID InstrumentRoutine(RTN rtn, VOID* args)
         RTN_Replace(rtn, (AFUNPTR) ariel_update_RTL_signals);
         fprintf(stderr,"Replacement complete.\n");
         return;
-
     } else if ((InterceptMemAllocations.Value() > 0) && RTN_Name(rtn) == "mlm_malloc") {
         // This means we want a special malloc to be used (needs a TLB map inside the virtual core)
         fprintf(stderr,"Identified routine: mlm_malloc, replacing with Ariel equivalent...\n");
@@ -1399,10 +1496,30 @@ int main(int argc, char *argv[])
 {
     if (PIN_Init(argc, argv)) return Usage();
 
+    syscall_id = PIN_CreateThreadDataKey(nullptr);
+    if (syscall_id == INVALID_TLS_KEY) {
+        std::cerr << "Failed to create thread-local storage key!" << std::endl;
+        return 1;
+    }
+    clone_is_mpi = PIN_CreateThreadDataKey(nullptr);
+    if (clone_is_mpi == INVALID_TLS_KEY) {
+        std::cerr << "Failed to create thread-local storage key!" << std::endl;
+        return 1;
+    }
+    _os_tid = PIN_CreateThreadDataKey(nullptr);
+    if (_os_tid == INVALID_TLS_KEY) {
+        std::cerr << "Failed to create thread-local storage key!" << std::endl;
+        return 1;
+    }
+    num_threads = 1;
+
+
     // Load the symbols ready for us to mangle functions.
     //PIN_InitSymbolsAlt(IFUNC_SYMBOLS);
     PIN_InitSymbols();
     PIN_AddFiniFunction(Fini, 0);
+    PIN_AddThreadStartFunction(ThreadStart, nullptr);
+    PIN_AddSyscallEntryFunction(SyscallEntry, 0);
 
     PIN_InitLock(&mainLock);
     PIN_InitLock(&mallocIndexLock);
@@ -1442,6 +1559,11 @@ int main(int argc, char *argv[])
     }
 
     core_count = MaxCoreCount.Value();
+
+    remap_id[0] = 0;
+    next_app_thread = 1;
+    next_mpi_thread = core_count;
+
     instrument_instructions = InstrumentInstructions.Value();
 
 // Pin version specific tunnel attach