Refactor parralelization code

src/Domains/BoxedDomain.cpp

@@ -1,5 +1,5 @@
 #include "BoxedDomain.h"
-#include "../LBM/parallel.h"
+#include "../LBM/parallel_bsp.h"
 
 namespace Domains {
 

src/Domains/DomainInitializer.cpp

@@ -4,7 +4,7 @@
 #include <math.h>       /* ceil */
 
 
-#include "../LBM/parallel.h"
+#include "../LBM/parallel_bsp.h"
 
 namespace Domains {
 
@@ -16,16 +16,8 @@ namespace Domains {
         d_domain_size(domainSize)
     {}
 
-    DomainInitializer::~DomainInitializer()
-    {}
-
     std::unique_ptr<Domain> DomainInitializer::domain()
     {
-        // setting the tagsize such that we can send the hash idx of a message
-        MCBSP_BYTESIZE_TYPE tag_size = sizeof(size_t);
-        bsp_set_tagsize(&tag_size);
-        bsp_sync();
-
         createNodes();
 
         std::unique_ptr<Domain> domain(new Domain);
@@ -38,29 +30,6 @@ namespace Domains {
 
         domain->set    = d_set;
         domain->omega = omega();
-
-        // setup messengers (for the parallelisation of the code)
-        bsp_sync();
-        // get the destination from bsp and apply it to the appropriate messenger
-        MCBSP_NUMMSG_TYPE nmessages = 0;
-        MCBSP_BYTESIZE_TYPE nbytes = 0;
-        bsp_qsize(&nmessages, &nbytes);
-        for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n)
-        {
-            size_t idx; // the hashIdx of the current messenger
-            MCBSP_BYTESIZE_TYPE status;
-            bsp_get_tag(&status,&idx);
-
-            if (status > 0)
-            {
-                size_t localIdx = 0;
-                bsp_move(&localIdx, status);
-                d_messengers[d_map_to_messenger[idx]].d_tag[0] = localIdx;
-            }
-            else
-                throw "Couldn't move the local idx during initialization phase.";
-        }
-
         domain->messengers = std::move(d_messengers);
 
         return domain;
@@ -100,6 +69,13 @@ namespace Domains {
             }
         }
 
+        // Connecting the nodes to their neighbours and set up the messengers
+
+        // setting the tagsize such that we can send the hash idx of a message
+        MCBSP_BYTESIZE_TYPE tag_size = sizeof(size_t);
+        bsp_set_tagsize(&tag_size);
+        bsp_sync();
+
         for (size_t idx = 0; idx < d_nodes.size(); ++idx)
             connectNodeToNeighbours(idx);
 
@@ -109,6 +85,8 @@ namespace Domains {
             size_t node_idx = d_messengers[idx].d_tag[0];
             d_nodes[node_idx].distributions[d_messengers[idx].d_tag[1]].neighbour = &d_messengers[idx].d_src;
         }
+
+        retrieveMessengers();
     }
 
     Node DomainInitializer::initializeNodeAt(std::vector<int> &position)
@@ -213,6 +191,31 @@ namespace Domains {
         // We don't need any post processors for this "dummy" domain
     }
 
+    void DomainInitializer::retrieveMessengers()
+    {
+        // setup messengers (for the parallelisation of the code)
+        bsp_sync();
+        // get the destination from bsp and apply it to the appropriate messenger
+        MCBSP_NUMMSG_TYPE nmessages = 0;
+        MCBSP_BYTESIZE_TYPE nbytes = 0;
+        bsp_qsize(&nmessages, &nbytes);
+        for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n)
+        {
+            size_t idx; // the hashIdx of the current messenger
+            MCBSP_BYTESIZE_TYPE status = 0;
+            bsp_get_tag(&status,&idx);
+
+            if (status > 0)
+            {
+                size_t localIdx = 0;
+                bsp_move(&localIdx, status);
+                d_messengers[d_map_to_messenger[idx]].d_tag[0] = localIdx;
+            }
+            else
+                throw "Couldn't move the local idx during initialization phase.";
+        }
+    }
+
     bool DomainInitializer::isInDomain(std::vector<int> &position)
     {
         return true;

src/Domains/DomainInitializer.h

@@ -34,7 +34,7 @@ namespace Domains {
 
         public:
             DomainInitializer(VelocitySet *set, std::vector<size_t> domainSize, size_t p = 0, size_t totalProcessors = 1);
-            virtual ~DomainInitializer();
+            virtual ~DomainInitializer() = default;
             std::unique_ptr<Domain> domain();
 
         protected:
@@ -46,6 +46,7 @@ namespace Domains {
             virtual bool isInDomain(std::vector<int> &position);
 
             virtual void createPostProcessors(std::vector<Node> &nodes);
+            virtual void retrieveMessengers();
 
             virtual double omega();
 

src/LBM/LBM_BSP_Program.cpp

@@ -0,0 +1,112 @@
+#include "LBM_BSP_Program.h"
+
+#include "../Domains/DomainInitializer.h"
+#include "../VelocitySets/d2q9.h"
+
+#include <ctime>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+
+using namespace Domains;
+using std::make_unique;
+using namespace mcbsp;
+
+size_t REPORT_PER_ITERATION = 10;
+
+LBM_BSP_Program::LBM_BSP_Program(size_t iterations, std::vector<size_t> domainSize)
+{
+    d_iterations = iterations;
+    d_domainSize = domainSize;
+}
+
+void LBM_BSP_Program::spmd() {
+    p = bsp_nprocs();
+    s = bsp_pid();
+
+    std::unique_ptr<LBM::Simulation> sim(setup_simulation());
+    run_simulation(sim.get());
+}
+
+
+BSP_program * LBM_BSP_Program::newInstance() {
+    return new LBM_BSP_Program(d_iterations, d_domainSize);
+}
+
+/**
+ *
+ * @param p amount of processors being used
+ * @param s current processor index
+ */
+LBM::Simulation * LBM_BSP_Program::setup_simulation()
+{
+    if (s == 0)
+        log_simulation_data();
+
+    bsp_sync();
+    double initialization_time = bsp_time();
+
+    auto set = new D2Q9;
+    DomainInitializer initializer(set, d_domainSize, s, p);
+
+    auto sim = new LBM::Simulation(initializer.domain());
+
+    // we'll send both the local idx and the direction which we we'll change
+    MCBSP_BYTESIZE_TYPE tag_size = sizeof(size_t[2]);
+    bsp_set_tagsize(&tag_size);
+
+    // Log initialization time and prepare computation time
+    bsp_sync();
+    double current_time = bsp_time();
+    if (s == 0)
+    {
+        std::ofstream out("logs/timings.log", std::ios::out | std::ios::app);
+        // Initialization time
+        out << "IT: " << (current_time - initialization_time) << " sec, ";
+    }
+    return sim;
+}
+
+/**
+ * Run ITERATIONS step and periodically report the current state of the simulation
+ * @param sim
+ */
+void LBM_BSP_Program::run_simulation(LBM::Simulation * sim)
+{
+    double process_time = bsp_time();
+
+    for (size_t iter = 0; iter < d_iterations; ++iter)
+    {
+        sim->step();
+
+        if (iter % REPORT_PER_ITERATION == 0)
+            sim->report();
+    }
+
+    // Create a timestamp
+    bsp_sync();
+    double current_time = bsp_time();
+    if (s == 0)
+    {
+        std::ofstream out("logs/timings.log", std::ios::out | std::ios::app);
+        // computation time
+        out << "CT: " << (current_time - process_time) << " sec" << '\n';
+    }
+}
+
+void LBM_BSP_Program::log_simulation_data()
+{
+    std::ofstream out("logs/timings.log", std::ios::out | std::ios::app);
+    size_t p = bsp_nprocs();
+    // Start by writing basic info to the file
+    out << "LBM, p: " << p << ", it: " << d_iterations << ", ds (";
+    show_vector(d_domainSize, out);
+    out << ", ";
+}
+
+void LBM_BSP_Program::show_vector(std::vector<size_t> vector, std::ofstream &out)
+{
+    for (size_t dim = 0; dim < (vector.size() - 1); ++dim)
+        out << vector[dim] << ", ";
+    out << vector[vector.size() - 1] << ")";
+}

src/LBM/LBM_BSP_Program.h

@@ -0,0 +1,40 @@
+#ifndef INCLUDED_LBM_BSP_PROGRAM
+#define INCLUDED_LBM_BSP_PROGRAM
+
+#include <cstdlib> // size_t
+#include <vector>
+#include <memory>
+
+#include "Simulation.h"
+
+#include "node.h"
+#include "parallel_bsp.h"
+
+class LBM_BSP_Program : public mcbsp::BSP_program {
+
+    size_t p;
+    size_t s;
+
+    size_t d_iterations;
+    std::vector<size_t> d_domainSize;
+
+    public:
+
+        LBM_BSP_Program(size_t iterations, std::vector<size_t> domainSize);
+
+    protected:
+
+        virtual void spmd();
+        virtual BSP_program * newInstance();
+
+    private:
+
+        LBM::Simulation * setup_simulation();
+        void run_simulation(LBM::Simulation * sim);
+        void log_simulation_data();
+        void show_vector(std::vector<size_t> vector, std::ofstream &out);
+
+};
+
+
+#endif