All Classes Namespaces Files Functions Variables Typedefs Enumerator Macros Groups Pages
MeterPU.h
Go to the documentation of this file.
1 
2 /*
3  * =====================================================================================
4  *
5  * Copyright (c) 2012 Linköping University
6  * Author: Lu Li <lu.li@liu.se>
7  *
8  * This program is free software: you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation, either version 3 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program. If not, see <http://www.gnu.org/licenses/>.
20  *
21  * =====================================================================================
22  */
23 
24 
25 
26 
27 #ifndef METERPU_INCLUSION_GUARD
28 #define METERPU_INCLUSION_GUARD
29 
30 //Headers
31 /*{{{*/
32 
33 /*
34  * =====================================================================================
35  *
36  * Filename: MeterPU.h
37  *
38  * Description: MeterPU: Library for measurement Abstraction
39  * Version: 0.8
40  *
41  * Author: Lu Li (lu.li@liu.se, explore.leo@gmail.com)
42  * Organization: IDA (www.ida.liu.se)
43  *
44  * =====================================================================================
45  */
46 
47 
48 //Module dependence
49 /*{{{*/
50 #ifdef ENABLE_SYSTEM_ENERGY //SYSTEM_ENERGY depends on PCM and NVML
51 
52 #ifndef ENABLE_PCM
53 #define ENABLE_PCM
54 #endif
55 
56 #ifndef ENABLE_NVML
57 #define ENABLE_NVML
58 #endif
59 
60 #endif
61 
62 #ifdef ENABLE_NVML
63 #include <nvml.h>
64 #endif
65 
66 #ifdef ENABLE_PCM
67 #include <cpucounters.h> //for PCM_Energy
68 #endif
69 /*}}}*/
70 
71 //Header files
72 /*{{{*/
73 #include <stdio.h>
74 #include <stdlib.h>
75 #include <pthread.h>
76 #include <vector>
77 #include <iostream>
78 #include <fstream>
79 #include <string>
80 #include <iomanip>
81 #include <unistd.h>
82 #include <assert.h>
83 /*}}}*/
84 
85 //Error message handling
86 /*{{{*/
87 #define PRINT_FUNC_NAME(message) std::cout<<"["<<__FILE__<<"#"<<__LINE__<<" "<<__func__<<"():] "<<message<<std::endl
88 #define PRINT_FUNC_NAME_CONT std::cout<<"["<<__FILE__<<"#"<<__LINE__<<" "<<__func__<<"():] "
89 
90 #define DECLARE_CLASS_NAME(name) static std::string class_name(){return name;}
91 #define PRINT_CLASS_FUNC_NAME(message) std::cout<<"["<<__FILE__<<"#"<<__LINE__<<" "<<class_name()<<" "<<__func__<<"():] "<<message<<std::endl
92 #define PRINT_CLASS_FUNC_NAME_CONT std::cout<<"["<<__FILE__<<"#"<<__LINE__<<" "<<class_name()<<" "<<__func__<<"():] "
93 /*}}}*/
94 
95 /*}}}*/
96 
97 //Doxygen Main page
98 /*{{{*/
120 /*}}}*/
121 
122 //Globals
123 /*{{{*/
124 //Macros
125 /*{{{*/
126 #define METERPU_TIME_MEASURE(x) clock_gettime(CLOCK_MONOTONIC, x)
127 /*}}}*/
128 
129 //Functions
130 /*{{{*/
131 
132 bool bash_exe (const std::string& cmd,std::vector<std::string>& out ) {
133  FILE* fp;
134  const int SIZEBUF = 1234;
135  char buf [SIZEBUF];
136  out = std::vector<std::string> ();
137  if ((fp = popen(cmd.c_str (), "r")) == NULL) {
138  return false;
139  }
140  std::string cur_string = "";
141  while (fgets(buf, sizeof (buf), fp)) {
142  cur_string += buf;
143  }
144  out.push_back (cur_string.substr (0, cur_string.size () - 1));
145  pclose(fp);
146  return true;
147 }
148 
149 /*}}}*/
150 
151 /*}}}*/
152 
153 //Implementation
154 /*{{{*/
155 namespace MeterPU
156 {
157 
158  //forward declaration
159  /*{{{*/
160 
161  //current group number: 7
162 
163 
164  template<class Type>
165  struct Meter_Traits;
166  struct Measurement_Controller;
167 
169  struct CPU_Time;
171 
172 #ifdef ENABLE_CUDA_TIME
174  struct CUDA_Time;
176 #endif
177 
178 #ifdef ENABLE_NVML
179  typedef unsigned int GPU_Device_Id_Type;
181  template<GPU_Device_Id_Type device_id=0>
182  struct NVML_Energy;
183  template<GPU_Device_Id_Type device_id>
185 #endif
186 
187 #ifdef ENABLE_PCM
189  struct PCM_Energy;
191 #endif
192 
193 #ifdef ENABLE_SYSTEM_ENERGY
195  template <GPU_Device_Id_Type... gpu_ids>
197  template<GPU_Device_Id_Type ...gpu_device_ids>
199 #endif
200 
201 
202  /*}}}*/
203 
204  //group0: Interface
205  /*{{{*/
221  template<class Type> class Meter
222  {
223  public:
225  {
228  }
240  void calc()
241  {
244  }
251  typename Meter_Traits<Type>::ResultType const &get_value() const {return meter_reading;}
259  //Remove it from release version
260 #ifdef METERPU_TEST
261  void test(){measurement_controller_object.test();}
262 #endif
263  private:
265 
270 
276  //measurement value.
281  };
282 
283 
284  // end of group0
286  /*}}}*/
287 
288  //group1: Traits
289  /*{{{*/
290 
291  //group1 Traits for Different Meter Type
292  /*{{{*/
298 
303  template<class Type> struct Meter_Traits
304  {
306 
309  typedef typename Type::Environment_Init_Type Environment_Init_Type;
311 
314  typedef typename Type::ResultType ResultType;
316 
319  typedef typename Type::Measurement_Controller Measurement_Controller;
320 
321  };
322 
324 
327  struct CPU_Time
328  {
330  typedef double ResultType;
333 
337  typedef struct timespec Time_Unit;
338  };
339 
340 
341 
347  inline CPU_Time::ResultType operator-(CPU_Time::Time_Unit const &stop_time,CPU_Time::Time_Unit const &start_time)
348  {
349  CPU_Time::ResultType elapse_time = ((stop_time.tv_sec - start_time.tv_sec)*1000000.0f+(stop_time.tv_nsec - start_time.tv_nsec)/1000.0f);
350 #ifdef DEBUG_OPERATOR_MINUS
351  if(elapse_time<0){
352  std::cout<<"stop_time.tv_sec: "<<stop_time.tv_sec<<"start_time.tv_sec: "<<start_time.tv_sec<<std::endl;
353  std::cout<<"stop_time.tv_nsec: "<<stop_time.tv_nsec<<"start_time.tv_nsec: "<<start_time.tv_nsec<<std::endl;
354  }
355 #endif
356  assert(elapse_time>=0);
357  return elapse_time;
358  }
364  inline bool operator<(CPU_Time::Time_Unit const &small_time,CPU_Time::Time_Unit const &large_time)
365  {
366  if(small_time.tv_sec<large_time.tv_sec)
367  return true;
368  if(small_time.tv_sec>large_time.tv_sec)
369  return false;
370  if(small_time.tv_nsec<large_time.tv_nsec)
371  return true;
372  return false;
373  }
379  inline bool operator==(CPU_Time::Time_Unit const &small_time,CPU_Time::Time_Unit const &large_time)
380  {
381  if(small_time.tv_sec!=large_time.tv_sec)
382  return false;
383  if(small_time.tv_nsec!=large_time.tv_nsec)
384  return false;
385  return true;
386  }
392  inline bool operator<=(CPU_Time::Time_Unit const &small_time,CPU_Time::Time_Unit const &large_time)
393  {
394  return (small_time<large_time || small_time == large_time);
395  }
396  //using namespace std::rel_ops;
397 
398 
399 #ifdef ENABLE_CUDA_TIME
400 
404  struct CUDA_Time
405  {
407  typedef float ResultType;
409  };
410 #endif
411 
412 #ifdef ENABLE_PCM
413 
417  struct PCM_Energy
418  {
420  typedef double ResultType;
422  //lu/ Type-specific
423  typedef double CPU_Energy_Type;
424  typedef double DRAM_Energy_Type;
425  };
426 #endif
427 
428 #ifdef ENABLE_NVML
429 
430 
431 
433 
437  template<GPU_Device_Id_Type device_id>
438  struct NVML_Energy
439  {
440  typedef double Energy_Unit;
441 
445 
446 
447  typedef struct timespec Time_Unit;
448  typedef unsigned int Power_Unit;
450 
453  typedef long double Hp_Power_Unit;
454  typedef std::vector<Time_Unit> Time_DB_Type;
455  typedef Time_DB_Type::const_iterator Time_DB_Const_Iterator_Type;
456  typedef std::vector<Power_Unit> Power_DB_Type;
457  typedef std::vector<Hp_Power_Unit> Hp_Power_DB_Type;
458  typedef Power_DB_Type::const_iterator Power_DB_Const_Iterator_Type;
459  typedef Hp_Power_DB_Type::const_iterator Hp_Power_DB_Const_Iterator_Type;
460 
461  };
462 #endif
463 
464 #ifdef ENABLE_SYSTEM_ENERGY
465 
470  template <GPU_Device_Id_Type... gpu_ids>
471  struct System_Energy
472  {
475  typedef double ResultType;
476  };
477 #endif
478 
479  // end of group1
481  /*}}}*/
482 
483  //Traits for container type
484  /*{{{*/
485 
486 #ifdef ENABLE_NVML
487  template <class Type>
489  {
490  static std::string header_message(){return Type::header_message();}
491  typedef typename Type::Data_Type Data_Type;
492  typedef typename Type::Const_Iterator_Type Const_Iterator_Type;
493  static void print(const Data_Type &a){
494  Const_Iterator_Type iterator;
495  std::cout<<"-----------------------------"<<"\n";
496  std::cout<<header_message()<<"\n";
497  std::cout<<"*****************************"<<"\n";
498  for(iterator=a.begin();iterator!=a.end();++iterator)
499  std::cout<<*iterator<<"\n";
500  std::cout<<"*****************************"<<"\n";
501  }
502  };
503 
505  {
506  static std::string header_message(){return "Power database:";}
509  };
510 
512  {
513  static std::string header_message(){return "Hp Power database:";}
516  };
517 
519  {
520  static std::string header_message(){return "Time database:";}
523  };
524  inline std::ostream &operator<<(std::ostream &out, NVML_Energy<>::Time_Unit ts)
525  {
526  out.precision(15);
527  out<<ts.tv_sec*1e6+ts.tv_nsec/1e3;
528  return out;
529  }
530 #endif
531 
532 
533  /*}}}*/
534 
535 
536  /*}}}*/
537 
538  //group2: Native Measurement Library Initializer
539  /*{{{*/
546 
551  {
552  virtual void init()=0;
553  };
554 
556 
560  {
561 /*{{{*/
562  private:
563  DECLARE_CLASS_NAME("CPU_Time_Environment_Init");
564  public:
565  void init(){
566 #if METERPU_VERBOSE >= 1
567  PRINT_CLASS_FUNC_NAME("called");
568 #endif
569  }
570 /*}}}*/
571  };
572 
573 #ifdef ENABLE_CUDA_TIME
574 
579  {
580 /*{{{*/
581  private:
582  DECLARE_CLASS_NAME("CUDA_Time_Environment_Init");
583  public:
584  void init(){
585 #if METERPU_VERBOSE >= 1
586  PRINT_CLASS_FUNC_NAME("called");
587 #endif
588  }
589 /*}}}*/
590  };
591 #endif
592 
593 
594 #ifdef ENABLE_PCM
595 
600  {
601 /*{{{*/
602  private:
603  DECLARE_CLASS_NAME("PCM_Energy_Environment_Init");
604  public:
605  void init(){
606 #if METERPU_VERBOSE >= 1
607  PRINT_CLASS_FUNC_NAME("called");
608 #endif
609  }
610 /*}}}*/
611  };
612 #endif
613 
614 #ifdef ENABLE_NVML
615 
620  {
621 /*{{{*/
622  public:
624  private:
626 
631  /*{{{*/
632  {
633 
634  public:
639  static void init_NVML()
640  {
641  static NVML_Manager instance;
642  }
643  private:
644  DECLARE_CLASS_NAME("NVML_Manager")
660  NVML_Manager(const NVML_Manager&);
664  inline void operator=(const NVML_Manager&);
668  void init()
669  {
670  nvmlReturn_t result;
671 
672  // Initialize NVML library
673  result = nvmlInit();
674  if (NVML_SUCCESS != result)
675  {
676  printf("Failed to initialize NVML: %s\n",
677  nvmlErrorString(result));
678 
679  printf("Press ENTER to continue...\n");
680  getchar();
681  exit(1);
682  }
683 #if METERPU_VERBOSE >= 1
684  PRINT_CLASS_FUNC_NAME("called");
685 #endif
686  }
690  void teardown()
691  {
692  nvmlReturn_t result;
693  result = nvmlShutdown();
694  if (NVML_SUCCESS != result)
695  {
696  printf("Failed to shutdown NVML: %s\n",
697  nvmlErrorString(result));
698 
699  printf("All done.\n");
700 
701  printf("Press ENTER to continue...\n");
702  getchar();
703  exit(1);
704  }
705 #if METERPU_VERBOSE >= 1
706  PRINT_CLASS_FUNC_NAME("called");
707 #endif
708  }
709  };
710  /*}}}*/
711  /*}}}*/
712  };
713 #endif
714 
715 #ifdef ENABLE_SYSTEM_ENERGY
716 
721  {
722 /*{{{*/
723  private:
724  DECLARE_CLASS_NAME("PCM_Energy_Environment_Init");
725  public:
726  void init(){
727 #if METERPU_VERBOSE >= 1
728  PRINT_CLASS_FUNC_NAME("called");
729 #endif
730  }
731 /*}}}*/
732  };
733 #endif
734 
735 
736  // end of group2
738  /*}}}*/
739 
740  //group3: Measurement controller
741  /*{{{*/
750 
755  {
761  virtual void start()=0;
767  virtual void stop()=0;
773  virtual void calc()=0;
780  virtual void show_meter_reading() const =0;
781  };
782 
784 
788  {
789  /*{{{*/
790  public:
793  CPU_Time::Time_Unit const &start_time_p,
794  CPU_Time::Time_Unit const &stop_time_p):start_time(start_time_p),stop_time(stop_time_p){}
795  private:
796  DECLARE_CLASS_NAME("CPU_Time_Measurement_Controller");
800  public:
801  void init()
802  {
803 #if METERPU_VERBOSE >= 1
804  PRINT_CLASS_FUNC_NAME("called");
805 #endif
806  }
807  inline void start(){
808 #if METERPU_VERBOSE >= 1
809  PRINT_CLASS_FUNC_NAME("called");
810 #endif
811 
812  //clock_gettime(CLOCK_MONOTONIC, &start_time);
814  }
815  inline void stop(){
816 #if METERPU_VERBOSE >= 1
817  PRINT_CLASS_FUNC_NAME("called");
818 #endif
819 
821  }
822  inline void calc(){
823 #if METERPU_VERBOSE >= 1
824  PRINT_CLASS_FUNC_NAME("called");
825 #endif
827 
828  }
836  {
837  return meter_reading;
838  }
839  void show_meter_reading() const {
840  std::cout<<"[CPU Time Meter] Time consumed is "<<get_value()<<" micro seconds."<<std::endl;
841  }
842  /*}}}*/
843  };
844 
845 #ifdef ENABLE_CUDA_TIME
846 
848 
852  {
853 /*{{{*/
854  private:
855  cudaEvent_t start_time;
856  cudaEvent_t stop_time;
858  public:
860  {
861  cudaEventCreate(&start_time);
862  cudaEventCreate(&stop_time);
863  }
865  {
866  cudaEventDestroy(start_time);
867  cudaEventDestroy(stop_time);
868  }
869  void init()
870  {
871 #if METERPU_VERBOSE >= 1
872  PRINT_CLASS_FUNC_NAME("called");
873 #endif
874  }
875  void start(){cudaEventRecord(start_time);}
876  void stop(){cudaEventRecord(stop_time);}
877  void calc(){cudaEventSynchronize(stop_time);
878  cudaEventElapsedTime(&meter_reading, start_time, stop_time);
879  meter_reading*=1000;
880  }
881  void show_meter_reading() const {
882  std::cout<<"[CUDA Time Meter] Time consumed is "<<get_value()<<" micro seconds."<<std::endl;
883  }
891 /*}}}*/
892  };
893 #endif
894 
895 
896 #ifdef ENABLE_PCM
897 
902  {
903  /*{{{*/
904  private:
905  DECLARE_CLASS_NAME("PCM_Energy_Measurement_Controller");
906  public:
907  void init(){}
908  void start(){
909 #if METERPU_VERBOSE >= 1
910  PRINT_CLASS_FUNC_NAME("called");
911 #endif
912  before_sstate = getSystemCounterState();
913  }
914  void stop(){
915 #if METERPU_VERBOSE >= 1
916  PRINT_CLASS_FUNC_NAME("called");
917 #endif
918  after_sstate = getSystemCounterState();
919  }
920  void calc()
921  {
924 #if METERPU_VERBOSE >= 2
925  PRINT_CLASS_FUNC_NAME_CONT<<"cpu_energy("<<cpu_energy<<")"<<std::endl;
926  PRINT_CLASS_FUNC_NAME_CONT<<"dram_energy("<<dram_energy<<")"<<std::endl;
927 #endif
929  }
937  void show_meter_reading() const{
938  std::cout<<"[CPU and DRAM Energy Meter] Energy consumed is: "<<get_value()<<" milli Joules."<<std::endl;
939  }
942  private:
947  void update_dram_energy(){dram_energy=getDRAMConsumedJoules(before_sstate, after_sstate);}
948  public:
949  PCM_Energy_Measurement_Controller():pcm(PCM::getInstance()){
950  if(pcm->program() != PCM::Success)
951  {
952  PRINT_CLASS_FUNC_NAME_CONT<<"PCM not initialized, exit.";
953  exit(1);
954  }
955  pcm->cleanup();
956  }
958 #if METERPU_VERBOSE >= 1
959  PRINT_CLASS_FUNC_NAME("called");
960 #endif
961  pcm->cleanup();}
962  private:
963  PCM *pcm;
964  SystemCounterState before_sstate;
965  SystemCounterState after_sstate;
966  /*}}}*/
967  };
968 #endif
969 
970 #ifdef ENABLE_NVML
971 
981  template<GPU_Device_Id_Type device_id_NVML_Energy_Measurement_Controller>
983  {
984 
985  /*{{{*/
986  private:
987  DECLARE_CLASS_NAME("NVML_Energy_Measurement_Controller");
988  public:
989 
991 
994  void init(){
995 /*{{{*/
999 /*}}}*/
1000  }
1001  private:
1002  template<GPU_Device_Id_Type device_id_value=device_id_NVML_Energy_Measurement_Controller>
1004  {
1005  /*{{{*/
1006  private:
1007  DECLARE_CLASS_NAME("NVML_Energy_Device_Init")
1009 
1014 
1017  nvmlDevice_t device;
1018  public:
1022  NVML_Energy_Device_Init():device_id(device_id_value){}
1030  void init()
1031  {
1032  nvmlReturn_t result;
1033 
1034  // Get the device handle
1035  result = nvmlDeviceGetHandleByIndex(device_id, &device);
1036 
1037  if (NVML_SUCCESS != result)
1038  {
1039  printf("Failed to get handle for device %i: %s\n",
1040  device_id, nvmlErrorString(result));
1041  exit(1);
1042  }
1043 
1044 #if METERPU_VERBOSE >= 1
1045 
1046  PRINT_CLASS_FUNC_NAME("called");
1047 #endif
1048  }
1052  const nvmlDevice_t &get_device() const{return device;}
1053  /*}}}*/
1054  };
1056  nvmlDevice_t device;
1064  }
1067  }
1069 
1070  public:
1071  void start(){
1072 /*{{{*/
1073 #if METERPU_VERBOSE >= 1
1074  PRINT_CLASS_FUNC_NAME("called");
1075 #endif
1076 
1077  //reset all states thus ready for next measurement
1078  meter_reading=0;
1079 
1081 
1083 /*}}}*/
1084  }
1085  void stop(){
1086 /*{{{*/
1087  record_stop_time();
1088 #if METERPU_VERBOSE >= 1
1089  PRINT_CLASS_FUNC_NAME("called");
1090 #endif
1091 
1092  //After the power curve becomes flat again, extend the flat region by 1000 microseconds,
1093  //to ensure there are more than 1 samples obtained. Substract the energy for that region later.
1094  //usleep(1000);
1096 
1097 #if METERPU_VERBOSE >= 3
1100 #endif
1101 /*}}}*/
1102  }
1103  void calc(){
1104 /*{{{*/
1105 #if METERPU_VERBOSE >= 1
1106  PRINT_CLASS_FUNC_NAME("called");
1107 #endif
1108  //persist_compensation_data();
1109 #ifdef DEBUG_OPERATOR_MINUS
1110  PRINT_CLASS_FUNC_NAME_CONT<<"power size: "<<sampling_thread_controller.get_power_db().size()<<std::endl;
1111 #endif
1112 
1113  switch(sampling_thread_controller.get_power_db().size())
1114  {
1115  case 0:
1116  meter_reading=0;
1117  break;
1118  case 1:
1119  //time: macro second, power: milli Watt
1120  //divided by 1e6, thus we get milli Joule
1122  * diff()/1e6;
1123  break;
1124  case 2:
1125  //power average multipled by time
1128  * diff()/2e6;
1129  break;
1130 
1131  default:
1132  //Dump original power data
1133 #ifdef METERPU_DUMP_DATA_FILE
1134  resetDir(); //TODO: calculate a path
1135  dumpTimeEvent();
1137 #endif
1138 
1139  //Do the postprocessing, only need to manipulate the power vector,
1140  //then everything later should just work
1141 
1142  //Remove power values that are too close
1143  //sampling_thread_controller.get_power_db() sampling_thread_controller.get_time_db
1145 
1146  //Correct power values based on Butcher's equation
1147  //Can use mapoverlap functional programming here
1150 
1151  //An optinal macro: dump power and time to file for visualization
1152 
1153  //Integrate all values
1154  //Substract the static energy when needed
1156 
1157  //dump corrected power data
1158 #ifdef METERPU_DUMP_DATA_FILE
1160 #endif
1161  }
1162 
1163  //Still need to call this method, it will reset the state of sampling_thread_controller for next measure.
1165 /*}}}*/
1166  }
1174  void show_meter_reading() const{ std::cout<<"[GPU Energy Meter] Energy consumed is: "<<get_value()<<" milli Joules."<<std::endl; }
1175 
1176 #if defined(METERPU_TEST)
1177  void test()
1178  {
1179 /*{{{*/
1180  //Common facility
1181  NVML_Energy<>::Time_Unit middle1,middle2,middle3,middle4,middle5;
1182  NVML_Energy<>::Time_Unit middle6,middle7,middle8,middle9,middle10;
1183  NVML_Energy<>::Time_Unit noise1,noise2,noise3;
1184  bool test_enable[]={0,0,0,0,0,1};
1185 
1186  /**********************************************************************************************/
1187  if(test_enable[0]){
1188 /*{{{*/
1192  sleep(1);
1193  sleep(1);
1194  record_stop_time();
1195 
1196 
1197 
1198  //test operator overloading
1199  std::cout<<"-----------------------------------------------------------"<<std::endl;
1200  std::cout<<"Test calc: 0 sample..."<<std::endl;
1201  calc();
1202  std::cout<<"Expected: "<<0<<std::endl;
1203  std::cout<<"Calculated: "<<get_value()<<std::endl;
1204  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1205 /*}}}*/
1206  }
1207 
1208  /**********************************************************************************************/
1209  if(test_enable[1]){
1210 /*{{{*/
1214  sleep(1);
1215  METERPU_TIME_MEASURE(&middle1);
1216  sleep(1);
1217  record_stop_time();
1218 
1219  sampling_thread_controller.get_time_db_nonconst().push_back(middle1);
1221 
1222 
1223  //test operator overloading
1224  std::cout<<"-----------------------------------------------------------"<<std::endl;
1225  std::cout<<"Test calc: 1 sample..."<<std::endl;
1226  calc();
1227  std::cout<<"Expected: "<<200<<std::endl;
1228  std::cout<<"Calculated: "<<get_value()<<std::endl;
1229  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1230 /*}}}*/
1231  }
1232  /**********************************************************************************************/
1233 
1234  if(test_enable[2]){
1235  /*{{{*/
1239  sleep(1);
1240  METERPU_TIME_MEASURE(&middle1);
1241  sleep(1);
1242  METERPU_TIME_MEASURE(&middle2);
1243  sleep(1);
1244  record_stop_time();
1245 
1246  sampling_thread_controller.get_time_db_nonconst().push_back(middle1);
1248  sampling_thread_controller.get_time_db_nonconst().push_back(middle2);
1250 
1251 
1252  //test operator overloading
1253  std::cout<<"-----------------------------------------------------------"<<std::endl;
1254  std::cout<<"Test calc: 2 sample..."<<std::endl;
1255  calc();
1256  std::cout<<"Expected: "<<300<<std::endl;
1257  std::cout<<"Calculated: "<<get_value()<<std::endl;
1258  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1259 
1260  /*}}}*/
1261  }
1262  /**********************************************************************************************/
1263 
1264  if(test_enable[3]){
1265  /*{{{*/
1269  sleep(1);
1270  METERPU_TIME_MEASURE(&middle1);
1271  sleep(1);
1272  METERPU_TIME_MEASURE(&middle2);
1273  sleep(1);
1274  METERPU_TIME_MEASURE(&middle3);
1275  sleep(1);
1276  record_stop_time();
1277 
1278  sampling_thread_controller.get_time_db_nonconst().push_back(middle1);
1280  sampling_thread_controller.get_time_db_nonconst().push_back(middle2);
1282  sampling_thread_controller.get_time_db_nonconst().push_back(middle3);
1284 
1285 
1286  //test operator overloading
1287  std::cout<<"-----------------------------------------------------------"<<std::endl;
1288  std::cout<<"Test calc: 3 sample..."<<std::endl;
1289  calc();
1290  std::cout<<"Expected: "<<400<<std::endl;
1291  std::cout<<"Calculated: "<<get_value()<<std::endl;
1292  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1293  /*}}}*/
1294  }
1295  /**********************************************************************************************/
1296 
1297  if(test_enable[4]){
1298  /*{{{*/
1302  sleep(1);
1303  METERPU_TIME_MEASURE(&middle1);
1304  METERPU_TIME_MEASURE(&noise1);
1305  sleep(1);
1306  METERPU_TIME_MEASURE(&middle2);
1307  METERPU_TIME_MEASURE(&noise2);
1308  sleep(1);
1309  METERPU_TIME_MEASURE(&middle3);
1310  METERPU_TIME_MEASURE(&noise3);
1311  sleep(1);
1312  record_stop_time();
1313 
1314  sampling_thread_controller.get_time_db_nonconst().push_back(middle1);
1316 
1319 
1320  sampling_thread_controller.get_time_db_nonconst().push_back(middle2);
1322 
1325 
1326  sampling_thread_controller.get_time_db_nonconst().push_back(middle3);
1328 
1331 
1332 
1333  //test operator overloading
1334  std::cout<<"-----------------------------------------------------------"<<std::endl;
1335  std::cout<<"Test calc: 3 samples with 3 noises..."<<std::endl;
1336  calc();
1337  std::cout<<"Expected: "<<400<<std::endl;
1338  std::cout<<"Calculated: "<<get_value()<<std::endl;
1339  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1340  /*}}}*/
1341  }
1342 
1343 
1344  if(test_enable[5]){
1345  /*{{{*/
1349  sleep(1);
1350  METERPU_TIME_MEASURE(&middle1);
1351  METERPU_TIME_MEASURE(&noise1);
1352  sleep(1);
1353  METERPU_TIME_MEASURE(&middle2);
1354  METERPU_TIME_MEASURE(&noise2);
1355  sleep(1);
1356  METERPU_TIME_MEASURE(&middle3);
1357  METERPU_TIME_MEASURE(&noise3);
1358  sleep(1);
1359  METERPU_TIME_MEASURE(&middle4);
1360  sleep(1);
1361  METERPU_TIME_MEASURE(&middle5);
1362  sleep(1);
1363  METERPU_TIME_MEASURE(&middle6);
1364  sleep(1);
1365  METERPU_TIME_MEASURE(&middle7);
1366  sleep(1);
1367  METERPU_TIME_MEASURE(&middle8);
1368  sleep(1);
1369  METERPU_TIME_MEASURE(&middle9);
1370  sleep(1);
1371  METERPU_TIME_MEASURE(&middle10);
1372  sleep(1);
1373  record_stop_time();
1374 
1375 
1376 
1377 
1378 
1379  sampling_thread_controller.get_time_db_nonconst().push_back(middle1);
1381 
1384 
1385  sampling_thread_controller.get_time_db_nonconst().push_back(middle2);
1387 
1390 
1391  sampling_thread_controller.get_time_db_nonconst().push_back(middle3);
1393 
1396 
1397  sampling_thread_controller.get_time_db_nonconst().push_back(middle4);
1399 
1400  sampling_thread_controller.get_time_db_nonconst().push_back(middle5);
1402 
1403  sampling_thread_controller.get_time_db_nonconst().push_back(middle6);
1405 
1406  sampling_thread_controller.get_time_db_nonconst().push_back(middle7);
1408 
1409  sampling_thread_controller.get_time_db_nonconst().push_back(middle8);
1411 
1412  sampling_thread_controller.get_time_db_nonconst().push_back(middle9);
1414 
1415  sampling_thread_controller.get_time_db_nonconst().push_back(middle10);
1417 
1418 
1419  //test operator overloading
1420  std::cout<<"-----------------------------------------------------------"<<std::endl;
1421  std::cout<<"Test calc: 10 samples, more realistic scenarios ..."<<std::endl;
1422  calc();
1423  std::cout<<"Expected: NA, check the database directly, if power changes drastically instead of gradually."<<std::endl;
1424  std::cout<<"Calculated: "<<get_value()<<std::endl;
1425  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1426  /*}}}*/
1427  }
1428  /**********************************************************************************************/
1429 /*}}}*/
1430  }
1431 #endif
1432  private:
1433  std::string path;
1434  void resetDir(){
1435  path="power_data-";
1436  std::vector<std::string> bash_results;
1437  if(!bash_exe("date +%F-%H-%M-%S", bash_results) ) {std::cout<<"Command date not executes!"<<std::endl;exit(1);}
1438 
1439  path+=*bash_results.begin();
1440  if ( system( ("mkdir "+path).c_str() ) ) {std::cout<<"Command mkdir not executes!"<<std::endl;exit(1);}
1441  }
1443  std::ofstream out( (path+"/time_event.csv").c_str() );
1444  if(!out){ std::cout<<"file not open!"<<std::endl;exit(1); }
1445  out<<"start_time_us,end_time_us"<<std::endl;
1446  out<<get_start_time()<<",";
1447  out<<get_stop_time()<<std::endl;
1448  }
1449  template <class T1, class T2>
1450  void dumpTwoVectors(const std::string &filename, const std::string &header, const T1 &keys, const T2 &values ) const {
1451  assert(keys.size() == values.size());
1452  std::ofstream out( filename.c_str() );
1453  if(!out){ std::cout<<"file not open!"<<std::endl;exit(1); }
1454  out<<header<<std::endl;
1455 
1456  typename T1::const_iterator it_keys;
1457  typename T2::const_iterator it_values;
1458 
1459  it_keys=keys.begin();
1460  it_values=values.begin();
1461 
1462  for(;it_keys!=keys.end();++it_keys,++it_values) {
1463  out<<*it_keys<<","<<*it_values<<std::endl;
1464  }
1465 
1466  }
1467  void dumpOriginalPowerData() const {
1468  dumpTwoVectors(path+"/original_power_data.csv", "time_us,power_mw", sampling_thread_controller.get_time_db(), sampling_thread_controller.get_power_db() );
1469  }
1471  dumpTwoVectors(path+"/corrected_power_data.csv", "time_us,power_mw", sampling_thread_controller.get_time_db(), correctedPowerDB );
1472  }
1474 /*{{{*/
1475 
1476 #ifdef METERPU_DEBUG_NVML_CALC
1477  std::cout<<"original databases: "<<std::endl;
1480  std::cout<<std::endl;
1481 #endif
1482 
1483  const CPU_Time::ResultType time_distance_us=time_distance_ms*1e3;
1484  //First mark the positions of non-redundant sample
1485  //Then add those samples to a new vector
1486  //Delete the redundant ones directly is not a
1487  //good idea, iterator will change, and
1488  //delete data in vector in general not good
1489  std::vector<bool> nonRedundantSampleFlag(sampling_thread_controller.get_power_db().size(), 0);
1490  //The first sample will always be non-redundant
1491  nonRedundantSampleFlag[0]=1;
1492 
1494  int index=1;
1495 
1496  //mark all elements except the first one
1497  while(it!=sampling_thread_controller.get_time_db().end()){
1498  if( *it - *(it-1) >= time_distance_us ){
1499  nonRedundantSampleFlag[index]=1;
1500  }
1501  ++index;
1502  ++it;
1503  }
1504 
1505  //make new vectors only contain nonredundant ones
1506  NVML_Energy<>::Time_DB_Type nonRedundantTimeDB;
1507  NVML_Energy<>::Power_DB_Type nonRedundantPowerDB;
1508 
1509  //placeholder at the starting position for time db
1510  nonRedundantTimeDB.push_back( NVML_Energy<>::Time_Unit() );
1511 
1512 
1513  index=0;
1514  while( index<nonRedundantSampleFlag.size() ) {
1515  if(nonRedundantSampleFlag[index]){
1516  nonRedundantTimeDB.push_back(sampling_thread_controller.get_time_db()[index]);
1517  nonRedundantPowerDB.push_back(sampling_thread_controller.get_power_db()[index]);
1518  }
1519  ++index;
1520  }
1521 
1522  //do the assignment
1523  sampling_thread_controller.set_time_db(nonRedundantTimeDB);
1524  sampling_thread_controller.set_power_db(nonRedundantPowerDB);
1525 
1526 
1527 #ifdef METERPU_DEBUG_NVML_CALC
1528  std::cout<<"After remove redundant samples"<<std::endl;
1531  std::cout<<std::endl;
1532 #endif
1533 
1534  /*}}}*/
1535  }
1538 /*{{{*/
1539  //P_true (t_i ) = P_meas (t_i ) + C × (P_meas (t_i+1 ) - P_meas (t_i-1 )) / (t_i+1 - t_i-1)
1540  //The first one and last one are harder to calculate,
1541  //since you don't have all data available,
1542  //placeholder at starting position for powerdb
1544  const float C = 0.84f;
1545  //start with the second power samples
1546  //corrected PowerDB: 0 x x ... //start with 2
1547  //meas PowerDB: x x ... //start with 1
1548  //timedb: 0 x x
1549  int index_placeholder=2, index=1;
1550  while( index < sampling_thread_controller.get_power_db().size() - 1 ) {
1551  //Only for debugging
1552  /*{{{*/
1553 #ifdef NEVER_DEFINE_IT_UNTIL_YOU_REALLY_WANT
1554  if(index==7){
1555  std::cout<<( (float)sampling_thread_controller.get_power_db()[index+1] - (float)sampling_thread_controller.get_power_db()[index-1])<<std::endl;
1556  std::cout<<(sampling_thread_controller.get_time_db()[index_placeholder+1] - sampling_thread_controller.get_time_db()[index_placeholder-1])/1e6<<std::endl;
1557  std::cout<<( (float)sampling_thread_controller.get_power_db()[index+1] - (float)sampling_thread_controller.get_power_db()[index-1])/((sampling_thread_controller.get_time_db()[index_placeholder+1] - sampling_thread_controller.get_time_db()[index_placeholder-1])/1e6)<<std::endl;
1558  std::cout<<C*( (float)sampling_thread_controller.get_power_db()[index+1] - (float)sampling_thread_controller.get_power_db()[index-1])/((sampling_thread_controller.get_time_db()[index_placeholder+1] - sampling_thread_controller.get_time_db()[index_placeholder-1])/1e6)<<std::endl;
1559  std::cout<<sampling_thread_controller.get_power_db()[index]+C*( (float)sampling_thread_controller.get_power_db()[index+1] - (float)sampling_thread_controller.get_power_db()[index-1])/((sampling_thread_controller.get_time_db()[index_placeholder+1] - sampling_thread_controller.get_time_db()[index_placeholder-1])/1e6)<<std::endl;
1560  }
1561 #endif
1562 /*}}}*/
1563  correctedPowerDB[index_placeholder] =
1565  C * ( (float)sampling_thread_controller.get_power_db()[index+1]
1566  - (float)sampling_thread_controller.get_power_db()[index-1] ) /
1567  ( (sampling_thread_controller.get_time_db()[index_placeholder+1]
1568  - sampling_thread_controller.get_time_db()[index_placeholder-1]) /1e6 );
1569  ++index;
1570  ++index_placeholder;
1571  }
1572 
1573  //Now original data is not useful
1575 
1576  //Make the first and last be the same as the
1577  //second and the second last as an approximation
1578  correctedPowerDB[1]
1579  = correctedPowerDB[2];
1580  *(correctedPowerDB.end() - 1)
1581  = *(correctedPowerDB.end() - 2);
1582 
1583 #ifdef METERPU_DEBUG_NVML_CALC
1584  std::cout<<"After correcting power samples"<<std::endl;
1585  //Container_Traits<Power_Vector_Container>::print(sampling_thread_controller.get_power_db());
1588  std::cout<<std::endl;
1589 #endif
1590  /*}}}*/
1591  }
1593 /*{{{*/
1594  //Fill the power value at kernel start time
1595  //*(sampling_thread_controller.get_power_db_nonconst().begin() )
1596  //= *(sampling_thread_controller.get_power_db().begin() + 1);
1598  //Fill the time value at kernel start time
1602 
1603 
1604  if( *(sampling_thread_controller.get_time_db().end()-1) < get_stop_time() ) {
1605  //Fill the time value at kernel end time
1607  //Fill the power value at kernel end time
1608  //sampling_thread_controller.get_power_db_nonconst().push_back(
1609  //*(sampling_thread_controller.get_power_db().end() - 1)
1610  //);
1611  correctedPowerDB.push_back( *(correctedPowerDB.end()-1) );
1612  }
1613 #ifdef METERPU_DEBUG_NVML_CALC
1614  std::cout<<"After filling values at start and end"<<std::endl;
1617  std::cout<<std::endl;
1618 #endif
1619 /*}}}*/
1620  }
1621  private:
1622 
1623 
1625  {
1626 /*{{{*/
1627 #if METERPU_VERBOSE >= 2 || defined(DEBUG_OPERATOR_MINUS)
1628  PRINT_CLASS_FUNC_NAME("called");
1629 #endif
1631 /*}}}*/
1632  }
1634  NVML_Energy<>::Hp_Power_DB_Type const &power_db,
1635  NVML_Energy<>::Time_DB_Type const &time_db
1636  )
1637  {
1638 /*{{{*/
1639 #if METERPU_VERBOSE >= 2 || defined(DEBUG_OPERATOR_MINUS)
1640  PRINT_CLASS_FUNC_NAME("called");
1641 #endif
1642  NVML_Energy<>::Energy_Unit totalArea=0;
1643  assert(power_db.size()==time_db.size());
1644  NVML_Energy<>::Hp_Power_DB_Type::const_iterator power_iter=power_db.begin();
1645  NVML_Energy<>::Time_DB_Type::const_iterator time_iter=time_db.begin();
1646  for(;power_iter!=power_db.end()-1;)
1647  {
1648  totalArea+=calAreaForOneTrapezoid(*time_iter,*(time_iter+1),*(power_iter),*(power_iter+1));
1649  ++power_iter;
1650  ++time_iter;
1651  }
1652  //Since time unit is micro second, here we convert it to second,
1653  //power unit is already milliwatts, thus we get milli Joule.
1654  return totalArea/1000000.0f;
1655 /*}}}*/
1656  }
1659  NVML_Energy<>::Time_Unit end_time,
1660  NVML_Energy<>::Hp_Power_Unit start_power,
1662  )
1663  {
1664 /*{{{*/
1665 #if METERPU_VERBOSE >= 2 || defined(DEBUG_OPERATOR_MINUS)
1666  PRINT_CLASS_FUNC_NAME("called");
1667 #endif
1668  NVML_Energy<>::Power_Unit h=(start_power>end_power?start_power:end_power);
1669  NVML_Energy<>::Power_Unit l=(start_power<=end_power?start_power:end_power);
1670  CPU_Time::ResultType r=end_time-start_time;
1671 #ifdef TEST_ENERGY_CAL
1672  std::cout<<(l*r+(h-l)*r/2)/1000000.0f<<std::endl;
1673 #endif
1674  return l*r+(h-l)*r/2;
1675 /*}}}*/
1676  }
1678 
1682  : public Measurement_Controller
1683  {
1684  /*{{{*/
1685  public:
1687  private:
1688  DECLARE_CLASS_NAME("Sampling_Thread_Controller");
1690  bool sampling;
1692 
1695  nvmlDevice_t device;
1697 
1702 
1706 
1711  static void *thread_program(void *arg)
1712  {
1713 #if METERPU_VERBOSE >= 1
1714  PRINT_CLASS_FUNC_NAME("start");
1715 #endif
1716 
1717  bool * const sampling_local=&((Sampling_Thread_Controller *)arg)->sampling;
1718  NVML_Energy<>::Time_Unit * const time_local=&((Sampling_Thread_Controller *)arg)->time;
1719  NVML_Energy<>::Power_Unit * const power_local=&((Sampling_Thread_Controller *)arg)->power;
1720  NVML_Energy<>::Time_DB_Type * const time_db_local=&((Sampling_Thread_Controller *)arg)->time_db;
1721  NVML_Energy<>::Power_DB_Type * const power_db_local=&((Sampling_Thread_Controller *)arg)->power_db;
1722  nvmlDevice_t const device_local=((Sampling_Thread_Controller *)arg)->device;
1723 
1724  nvmlReturn_t return_code;
1725 #if METERPU_VERBOSE >= 2
1726  PRINT_CLASS_FUNC_NAME_CONT<<"sampling_local("<<*sampling_local<<")\n";
1727 #endif
1728 
1729  while(*sampling_local)
1730  {
1731 
1732  //Sampling power
1733  return_code = nvmlDeviceGetPowerUsage( device_local, power_local );
1734  if ( return_code != NVML_SUCCESS) {
1735  continue;
1736  }
1737  METERPU_TIME_MEASURE(time_local);
1738 
1739 #if METERPU_VERBOSE >= 4
1740  PRINT_CLASS_FUNC_NAME_CONT<<"current power("<<*power_local<<")\n";
1741 #endif
1742 
1743  //Add time and power pair to database
1744  power_db_local->push_back(*power_local);
1745  time_db_local->push_back(*time_local);
1746  }
1747 
1748 #if METERPU_VERBOSE >= 2
1749  NVML_Energy<>::Time_DB_Type::iterator it=time_db_local->end();
1750  PRINT_CLASS_FUNC_NAME_CONT<<"Obtained stop signal, the last time stamp of sampling is: "
1751  <<*--it
1752  <<std::endl;
1753 #endif
1754 
1755  pthread_exit((void*) arg);
1756  }
1757  pthread_t thread;
1758  pthread_attr_t attr;
1759  public:
1760  void start()
1761  {
1762  int rc;
1763 
1764  pthread_attr_init(&attr);
1765  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
1766 
1767  //Set the thread runnable flag
1768  sampling=true;
1769 
1770 #if METERPU_VERBOSE >= 2
1771  PRINT_CLASS_FUNC_NAME_CONT<<"sampling("<<sampling<<")\n";
1772 #endif
1773 
1774  //Release the thread
1775  rc = pthread_create(&thread, &attr, thread_program, (void *)this);
1776 #if METERPU_VERBOSE >= 1
1777  PRINT_CLASS_FUNC_NAME("thread released");
1778 #endif
1779  if (rc) {
1780  printf("ERROR; return code from pthread_create() is %d\n", rc);
1781  exit(-1);
1782  }
1783  }
1784  void stop(){
1785  int rc;
1786  void *status;
1787 
1788  //Set stop signal
1789  sampling=false;
1790 
1791  //Wait for thread to join
1792  rc = pthread_join(thread, &status);
1793  if (rc) {
1794  printf("ERROR; return code from pthread_join() is %d\n", rc);
1795  exit(-1);
1796  }
1797  pthread_attr_destroy(&attr);
1798 
1799 #if METERPU_VERBOSE >= 1
1800 
1802  CPU_Time_Measurement_Controller temp(*time_db.begin(),*--it);
1803  temp.calc();
1804  PRINT_CLASS_FUNC_NAME_CONT<<"time period of sampling is: "
1805  <<temp.get_value()
1806  <<std::endl;
1807  PRINT_CLASS_FUNC_NAME_CONT<<"number of sampling is: "
1808  <<power_db.size()
1809  <<std::endl;
1810  PRINT_CLASS_FUNC_NAME_CONT<<"The first sample of power is: "
1811  <<*power_db.begin()
1812  <<std::endl;
1813  PRINT_CLASS_FUNC_NAME_CONT<<"sampling rate is: "
1814  <<power_db.size()/temp.get_value()
1815  <<" #samples/sec."
1816  <<std::endl;
1817 #endif
1818 
1819 
1820  }
1821  void calc(){
1822 #if METERPU_VERBOSE >= 1
1823  PRINT_CLASS_FUNC_NAME("called");
1824 #endif
1825 
1826  //reset state for next measurement
1827  reset_state();
1828 
1829  }
1830  void show_meter_reading() const {PRINT_FUNC_NAME_CONT<<"Not applicable to call me..."<<std::endl;}
1831  void set_device(const nvmlDevice_t &device_handle){device=device_handle;}
1832  const NVML_Energy<>::Power_Unit &get_power() const {return power;}
1840  assert(time_db.size()>=1);
1842  return *--it;
1843  }
1845  assert(time_db.size()>=1);
1846  return *time_db.begin();
1847  }
1848  void reset_state(){time_db.clear();power_db.clear();}
1849 
1850  /*}}}*/
1851  };
1853  /*}}}*/
1854  };
1855 
1856 #endif
1857 
1858 
1859 //Obsolete
1860 #ifdef ENABLE_NVML_OLD
1861  template<GPU_Device_Id_Type device_id_NVML_Energy_Measurement_Controller>
1863  {
1864  /*{{{*/
1865  private:
1866  DECLARE_CLASS_NAME("NVML_Energy_Measurement_Controller");
1867  public:
1868  void init(){
1872  }
1873  private:
1874  template<GPU_Device_Id_Type device_id_value=device_id_NVML_Energy_Measurement_Controller>
1875  struct NVML_Energy_Device_Init
1876  {
1877  /*{{{*/
1878  private:
1879  DECLARE_CLASS_NAME("NVML_Energy_Device_Init")
1881 
1886 
1889  nvmlDevice_t device;
1890  public:
1894  NVML_Energy_Device_Init():device_id(device_id_value){}
1902  void init()
1903  {
1904  nvmlReturn_t result;
1905 
1906  // Get the device handle
1907  result = nvmlDeviceGetHandleByIndex(device_id, &device);
1908 
1909  if (NVML_SUCCESS != result)
1910  {
1911  printf("Failed to get handle for device %i: %s\n",
1912  device_id, nvmlErrorString(result));
1913  exit(1);
1914  }
1915 
1916 #if METERPU_VERBOSE >= 1
1917 
1918  PRINT_CLASS_FUNC_NAME("called");
1919 #endif
1920  }
1924  const nvmlDevice_t &get_device() const{return device;}
1925  /*}}}*/
1926  };
1927  NVML_Energy_Device_Init<> nvml_energy_device_init;
1928  nvmlDevice_t device;
1929  NVML_Energy<>::Time_Unit execution_stop_time;
1930  typename NVML_Energy<>::Energy_Unit meter_reading;
1931  public:
1932 #ifdef METERPU_TEST
1933  void test()
1934  /*{{{*/
1935  {
1936  //Common facility
1937  NVML_Energy<>::Time_Unit start_time, stop_time, compensation_end;
1938  //clock_gettime(CLOCK_MONOTONIC, &start_time);
1939  METERPU_TIME_MEASURE(&start_time);
1940  sleep(1);
1941  //clock_gettime(CLOCK_MONOTONIC, &stop_time);
1942  METERPU_TIME_MEASURE(&stop_time);
1943  sleep(1);
1944  //clock_gettime(CLOCK_MONOTONIC, &compensation_end);
1945  METERPU_TIME_MEASURE(&compensation_end);
1946 
1947 
1948  //test operator overloading
1949  std::cout<<"-----------------------------------------------------------"<<std::endl;
1950  std::cout<<"Test operator overloading..."<<std::endl;
1951  std::cout<<"Expected: "<<1e6<<std::endl;
1952  std::cout<<"Calculated: "<<stop_time-start_time<<std::endl;
1953  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1954  //A failure case
1955  //std::cout<<start_time-stop_time<<std::endl;
1956 
1957  //test calAreaForOneTrapezoid()
1958  std::cout<<"-----------------------------------------------------------"<<std::endl;
1959  std::cout<<"Test calAreaForOneTrapezoid()..."<<std::endl;
1960  std::cout<<"Expected: "<<1.5<<std::endl;
1961  std::cout<<"Calculated: "<<calAreaForOneTrapezoid(start_time,stop_time,1,2)<<std::endl;
1962  std::cout<<"Expected: "<<1.5<<std::endl;
1963  std::cout<<"Calculated: "<<calAreaForOneTrapezoid(start_time,stop_time,2,1)<<std::endl;
1964  std::cout<<"Expected: "<<1<<std::endl;
1965  std::cout<<"Calculated: "<<calAreaForOneTrapezoid(start_time,stop_time,1,1)<<std::endl;
1966  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1967 
1968  //test calTotalArea()
1969  std::cout<<"-----------------------------------------------------------"<<std::endl;
1970  std::cout<<"Test calTotalArea()..."<<std::endl;
1971  NVML_Energy<>::Power_DB_Type power_db;
1972  power_db.push_back(1);
1973  power_db.push_back(2);
1974  power_db.push_back(3);
1975 
1976  NVML_Energy<>::Time_DB_Type time_db;
1977  time_db.push_back(start_time);
1978  time_db.push_back(stop_time);
1979  time_db.push_back(compensation_end);
1980 
1981  std::cout<<"Expected: "<<4<<std::endl;
1982  std::cout<<"Calculated: "<<calTotalArea(power_db,time_db)<<std::endl;
1983  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1984 
1985  //test if energy calculation is correct by cooperation of calTotalArea(), calCompensatedStaticEnergy()
1986  //and calStaticEnergy()
1987  std::cout<<"-----------------------------------------------------------"<<std::endl;
1988  std::cout<<"Test energy calculation by"
1989  <<" calTotalArea(), calCompensatedStaticEnergy() and calStaticEnergy()..."<<std::endl;
1990  std::cout<<"Expected: "<<3<<std::endl;
1991  std::cout<<"Calculated: "
1992  <<calTotalArea(power_db,time_db)-calCompensatedStaticEnergy(stop_time,compensation_end,1)<<std::endl;
1993  std::cout<<"Expected: "<<2<<std::endl;
1994  std::cout<<"Calculated: "
1995  <<calTotalArea(power_db,time_db)-calCompensatedStaticEnergy(stop_time,compensation_end,1)-calStaticEnergy(start_time,stop_time,1)<<std::endl;
1996  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
1997 
1998  //test time unit comparison
1999  std::cout<<"-----------------------------------------------------------"<<std::endl;
2000  std::cout<<"Test time unit comparison"<<std::endl;
2001  std::cout<<std::boolalpha;
2002  CPU_Time::Time_Unit small, large;
2003  //Equal case
2004  small.tv_sec=1;small.tv_nsec=2;
2005  large.tv_sec=1;large.tv_nsec=2;
2006  std::cout<<"1 Expected: equal true"<<std::endl;
2007  std::cout<<"1 Calculated: "<<(small==large)<<std::endl;
2008  std::cout<<"2 Expected: small and equal true"<<std::endl;
2009  std::cout<<"2 Calculated: "<<(small<=large)<<std::endl;
2010  //First small
2011  small.tv_sec=2;small.tv_nsec=2;
2012  large.tv_sec=3;large.tv_nsec=2;
2013  std::cout<<"3 Expected: small true"<<std::endl;
2014  std::cout<<"3 Calculated: "<<(small<large)<<std::endl;
2015  std::cout<<"4 Expected: small and equal true"<<std::endl;
2016  std::cout<<"4 Calculated: "<<(small<=large)<<std::endl;
2017  //Second small
2018  small.tv_sec=1;small.tv_nsec=1;
2019  large.tv_sec=1;large.tv_nsec=2;
2020  std::cout<<"5 Expected: small true"<<std::endl;
2021  std::cout<<"5 Calculated: "<<(small<large)<<std::endl;
2022  std::cout<<"6 Expected: small and equal true"<<std::endl;
2023  std::cout<<"6 Calculated: "<<(small<=large)<<std::endl;
2024  //Fisrt small, second large
2025  small.tv_sec=0;small.tv_nsec=4;
2026  large.tv_sec=1;large.tv_nsec=2;
2027  std::cout<<"7 Expected: small true"<<std::endl;
2028  std::cout<<"7 Calculated: "<<(small<large)<<std::endl;
2029  std::cout<<"8 Expected: small and equal true"<<std::endl;
2030  std::cout<<"8 Calculated: "<<(small<=large)<<std::endl;
2031  //Fisrt large, second small
2032  small.tv_sec=2;small.tv_nsec=1;
2033  large.tv_sec=1;large.tv_nsec=2;
2034  std::cout<<"9 Expected: small false"<<std::endl;
2035  std::cout<<"9 Calculated: "<<(small<large)<<std::endl;
2036  std::cout<<"10 Expected: small and equal false"<<std::endl;
2037  std::cout<<"10 Calculated: "<<(small<=large)<<std::endl;
2038  std::cout<<"-----------------------------------------------------------"<<std::endl<<std::endl;
2039  }
2040  /*}}}*/
2041 #endif
2042 
2049  void start(){
2050 #if METERPU_VERBOSE >= 1
2051  PRINT_CLASS_FUNC_NAME("called");
2052 #endif
2053 
2054  //reset all states thus ready for next measurement
2055  meter_reading=0;
2056 
2057  sampling_static_power();
2059  }
2060 
2066  void stop(){
2067 #if METERPU_VERBOSE >= 1
2068  PRINT_CLASS_FUNC_NAME("called");
2069 #endif
2070 
2071 #if METERPU_VERBOSE >= 2
2072  PRINT_CLASS_FUNC_NAME("mark down the execution stop time");
2073 #endif
2074  //mark the execution stop time
2075  //clock_gettime(CLOCK_MONOTONIC, &execution_stop_time);
2076  METERPU_TIME_MEASURE(&execution_stop_time);
2077 #if METERPU_VERBOSE >= 2
2078  PRINT_CLASS_FUNC_NAME_CONT<<"time stamp of execution time: ("<<execution_stop_time<<")"<<std::endl;
2079 #endif
2080 
2081  typename NVML_Energy<>::Hp_Power_Unit power_threshold
2082  = //(typename NVML_Energy<>::Power_Unit)
2083  (static_power_value*1.05f);
2084 
2085 
2086  do{
2087 #if METERPU_VERBOSE >= 3
2088  PRINT_CLASS_FUNC_NAME_CONT<<"waiting to release stop signal: power("<<sampling_thread_controller.get_power()<<")\n";
2089 #endif
2090  }while(sampling_thread_controller.get_power()>power_threshold);
2091 #if METERPU_VERBOSE >= 2
2092  PRINT_CLASS_FUNC_NAME_CONT<<"power_threshold("<<power_threshold<<")\n";
2093  PRINT_CLASS_FUNC_NAME_CONT<<"about to release stop signal: power("<<sampling_thread_controller.get_power()<<")\n";
2094 #endif
2095  //After the power curve becomes flat again, extend the flat region by 1000 microseconds,
2096  //to ensure there are more than 1 samples obtained. Substract the energy for that region later.
2097  //usleep(1000);
2099 
2100 #if METERPU_VERBOSE >= 3
2103 #endif
2104 
2105 
2106 
2107  }
2108  void calc(){
2109 #if METERPU_VERBOSE >= 1
2110  PRINT_CLASS_FUNC_NAME("called");
2111 #endif
2112  //persist_compensation_data();
2113 #ifdef DEBUG_OPERATOR_MINUS
2114  PRINT_CLASS_FUNC_NAME_CONT<<"power size: "<<sampling_thread_controller.get_power_db().size()<<std::endl;
2115 #endif
2116 
2117  //Do the calculation here, and assign them to meter_reading,
2118  //offer two possibilities, whole energy and dynamic energy.
2119  switch(sampling_thread_controller.get_power_db().size())
2120  {
2121  case 0:
2122  meter_reading=0;
2123  break;
2124  case 1:
2125  //The energy estimated here is the half of the energy that minimum time needed
2126  //to obtain two samples when execute nothing.
2127  //which is 5.241385
2128  meter_reading=5.241385;
2129  break;
2130  default:
2131  //Substract the static energy when needed
2132  meter_reading=calTotalArea()-calCompensatedStaticEnergy();//-calStaticEnergy();
2133  }
2134 
2135  //Still need to call this method, it will reset the state of sampling_thread_controller for next measure.
2137 
2138  //Let the GPU cool down for 10 seconds to mitigate temperature effect.
2139  //sleep(1);
2140 
2141  }
2142  NVML_Energy<>::Energy_Unit get_value() const{return meter_reading;}
2143  void show_meter_reading() const{
2144  std::cout<<"Energy consumed is: "<<get_value()<<" milli Joules."<<std::endl;
2145  }
2146  private:
2150  NVML_Energy<>::Energy_Unit calTotalArea()
2151  {
2152 #if METERPU_VERBOSE >= 2 || defined(DEBUG_OPERATOR_MINUS)
2153  PRINT_CLASS_FUNC_NAME("called");
2154 #endif
2156  }
2160  NVML_Energy<>::Energy_Unit calTotalArea(
2161  NVML_Energy<>::Power_DB_Type const &power_db,
2162  NVML_Energy<>::Time_DB_Type const &time_db
2163  )
2164  {
2165 #if METERPU_VERBOSE >= 2 || defined(DEBUG_OPERATOR_MINUS)
2166  PRINT_CLASS_FUNC_NAME("called");
2167 #endif
2168  NVML_Energy<>::Energy_Unit totalArea=0;
2169  //std::cout<<std::endl<<power_db.size()<<std::endl;
2170  assert(power_db.size()==time_db.size());
2171  NVML_Energy<>::Power_DB_Type::const_iterator power_iter=power_db.begin();
2172  NVML_Energy<>::Time_DB_Type::const_iterator time_iter=time_db.begin();
2173  for(;power_iter!=power_db.end()-1;)
2174  {
2175  totalArea+=calAreaForOneTrapezoid(*time_iter,*(time_iter+1),*(power_iter),*(power_iter+1));
2176  ++power_iter;
2177  ++time_iter;
2178  }
2179  //Since time unit is micro second, here we convert it to second,
2180  //power unit is already milliwatts, thus we get milli Joule.
2181  return totalArea/1000000.0f;
2182  }
2186  NVML_Energy<>::Energy_Unit calAreaForOneTrapezoid(
2187  NVML_Energy<>::Time_Unit start_time,
2188  NVML_Energy<>::Time_Unit end_time,
2189  NVML_Energy<>::Power_Unit start_power,
2190  NVML_Energy<>::Power_Unit end_power
2191  )
2192  {
2193 #if METERPU_VERBOSE >= 2 || defined(DEBUG_OPERATOR_MINUS)
2194  PRINT_CLASS_FUNC_NAME("called");
2195 #endif
2196  NVML_Energy<>::Power_Unit h=(start_power>end_power?start_power:end_power);
2197  NVML_Energy<>::Power_Unit l=(start_power<=end_power?start_power:end_power);
2198  CPU_Time::ResultType r=end_time-start_time;
2199 #ifdef TEST_ENERGY_CAL
2200  //PRINT_CLASS_FUNC_NAME_CONT<<"AreaForOneTrapezoid("<<l*r+(h-l)*r/2<<")"<<std::endl;
2201  std::cout<<(l*r+(h-l)*r/2)/1000000.0f<<std::endl;
2202 #endif
2203  return l*r+(h-l)*r/2;
2204  }
2209  NVML_Energy<>::Energy_Unit calCompensatedStaticEnergy()
2210  {
2211 #if METERPU_VERBOSE >= 2 || defined(DEBUG_OPERATOR_MINUS)
2212  PRINT_CLASS_FUNC_NAME("called");
2213 #endif
2214  return calCompensatedStaticEnergy(execution_stop_time,sampling_thread_controller.get_stop_time(),static_power_value);
2215  }
2220  NVML_Energy<>::Energy_Unit calCompensatedStaticEnergy(
2221  NVML_Energy<>::Time_Unit execution_end,
2222  NVML_Energy<>::Time_Unit compensation_end,
2223  NVML_Energy<>::Power_Unit static_power_value
2224  )
2225  {
2226  CPU_Time::ResultType compensation_period=(execution_end<compensation_end?compensation_end-execution_end:0);
2227 #if METERPU_VERBOSE>=2 || defined TEST_ENERGY_CAL || defined DEBUG_OPERATOR_MINUS
2228  PRINT_CLASS_FUNC_NAME_CONT<<" compensation_end: "<<compensation_end<<std::endl;
2229  PRINT_CLASS_FUNC_NAME_CONT<<" execution_end: "<<execution_end<<std::endl;
2230  PRINT_CLASS_FUNC_NAME_CONT<<"compensation_period("<<(compensation_end-execution_end)/1000000.0f<<")"<<std::endl;
2231 
2232 #endif
2233  assert(compensation_period>=0);
2234  return (compensation_period)*static_power_value/1000000.0f;
2235  }
2239  NVML_Energy<>::Energy_Unit calStaticEnergy()
2240  {
2241  return calStaticEnergy(sampling_thread_controller.get_start_time(),execution_stop_time,static_power_value);
2242  }
2246  NVML_Energy<>::Energy_Unit calStaticEnergy(
2247  NVML_Energy<>::Time_Unit execution_start,
2248  NVML_Energy<>::Time_Unit execution_end,
2249  NVML_Energy<>::Power_Unit static_power_value
2250  )
2251  {
2252  return (execution_end-execution_start)*static_power_value/1000000.0f;
2253  }
2254 
2255  //lu/ TODO: might be a good place to apply smart sampling by std_dev
2256  void sampling_static_power(){
2257  /*{{{*/
2258  nvmlReturn_t return_code;
2259 
2260  static_power_value=0;
2261  int valid_power_sampling_count=0;
2262  const int num_of_samples=10;
2263 
2264  for(int i=0;i<num_of_samples;++i){
2265  //Sampling power
2266  return_code = nvmlDeviceGetPowerUsage( device, &power );
2267  if ( return_code != NVML_SUCCESS) {
2268 #if METERPU_VERBOSE >= 3
2269  PRINT_CLASS_FUNC_NAME("warnings(failed in getting one power sample value)");
2270 #endif
2271  --i;
2272  continue;
2273  }
2274 
2275 #if METERPU_VERBOSE >= 3
2276  PRINT_CLASS_FUNC_NAME_CONT<<"current sample value("<<power<<")\n";
2277 #endif
2278 
2279  valid_power_sampling_count++;
2280  static_power_value+=power;
2281  }
2282 #if METERPU_VERBOSE >= 2 || defined TEST_ENERGY_CAL
2283  PRINT_CLASS_FUNC_NAME_CONT<<"Sum of static power value("<<static_power_value<<")\n";
2284  PRINT_CLASS_FUNC_NAME_CONT<<"valid_power_sampling_count("<<valid_power_sampling_count<<")\n";
2285 #endif
2286  static_power_value=static_power_value/valid_power_sampling_count;
2287 #if METERPU_VERBOSE >= 2
2288  std::cout<<std::setprecision(10)<<"sampling_static_power(): final static power("<<static_power_value<<")\n";
2289 #endif
2290  /*}}}*/
2291  }
2293 
2295  NVML_Energy<>::Hp_Power_Unit static_power_value;
2296  NVML_Energy<>::Power_Unit power;
2297  //void persist_compensation_data(){
2298 
2299  //#if METERPU_VERBOSE >= 1
2300  //PRINT_CLASS_FUNC_NAME("called");
2301  //#endif
2302 
2306  //std::ofstream out1("appgen_energy_compensation.csv");
2307  //if(!out1){
2308  //std::cout<<"File creation error!"<<std::endl;
2309  //exit(1);
2310  //}
2311  //out1<<"\"diff_tvsec\",\"diff_tvnsec\",\"static_power\""<<std::endl;
2312 
2313  //#if METERPU_VERBOSE >= 2
2314  //PRINT_CLASS_FUNC_NAME_CONT<<"execution_stop_time: "<<execution_stop_time<<std::endl;
2315  //PRINT_CLASS_FUNC_NAME_CONT<<"sampling stop time: "<<sampling_thread_controller.get_stop_time()<<std::endl;
2316  //#endif
2317 
2318  //time_t diff_sec=sampling_thread_controller.get_stop_time().tv_sec-execution_stop_time.tv_sec;
2319  //long diff_nsec=sampling_thread_controller.get_stop_time().tv_nsec-execution_stop_time.tv_nsec;
2320  //long long diff_compensation_period=diff_sec*(long long)1000000000+diff_nsec;
2321  //#if METERPU_VERBOSE >= 2
2322  //PRINT_CLASS_FUNC_NAME_CONT<<"diff_sec("<<diff_sec<<")"<<std::endl;
2323  //PRINT_CLASS_FUNC_NAME_CONT<<"diff_nsec("<<diff_nsec<<")"<<std::endl;
2324  //PRINT_CLASS_FUNC_NAME_CONT<<"diff_compensation_period("<<diff_compensation_period<<")"<<std::endl;
2325  //#endif
2326  //if(diff_compensation_period<0)
2327  //PRINT_CLASS_FUNC_NAME_CONT<<"Warning: Sampling stopped earlier than execution ends."<<std::endl;
2328  //out1<<diff_sec<<","<<diff_nsec<<","<<static_power_value<<std::endl;
2329 
2330  //}
2331 
2332  class Sampling_Thread_Controller
2333  : public Measurement_Controller
2334  {
2335  /*{{{*/
2336  public:
2338  private:
2339  DECLARE_CLASS_NAME("Sampling_Thread_Controller");
2340  //lu/ A good encapsulation to define a constant
2341  static NVML_Energy<>::Power_Unit get_UNREALISTIC_POWER_VALUE(){ return (NVML_Energy<>::Power_Unit)100000000;}
2342  /*Static bool sampling;*/
2343  bool sampling;
2345 
2348  nvmlDevice_t device;
2350 
2353  NVML_Energy<>::Power_Unit power;
2355 
2358  NVML_Energy<>::Time_Unit time;
2360 
2363  NVML_Energy<>::Time_DB_Type time_db;
2364  NVML_Energy<>::Power_DB_Type power_db;
2365  static void *thread_program(void *arg)
2366  {
2367 #if METERPU_VERBOSE >= 1
2368  PRINT_CLASS_FUNC_NAME("start");
2369 #endif
2370 
2371  bool * const sampling_local=&((Sampling_Thread_Controller *)arg)->sampling;
2372  NVML_Energy<>::Time_Unit * const time_local=&((Sampling_Thread_Controller *)arg)->time;
2373  NVML_Energy<>::Power_Unit * const power_local=&((Sampling_Thread_Controller *)arg)->power;
2374  NVML_Energy<>::Time_DB_Type * const time_db_local=&((Sampling_Thread_Controller *)arg)->time_db;
2375  NVML_Energy<>::Power_DB_Type * const power_db_local=&((Sampling_Thread_Controller *)arg)->power_db;
2376  nvmlDevice_t const device_local=((Sampling_Thread_Controller *)arg)->device;
2377 
2378  nvmlReturn_t return_code;
2379 #if METERPU_VERBOSE >= 2
2380  PRINT_CLASS_FUNC_NAME_CONT<<"sampling_local("<<*sampling_local<<")\n";
2381 #endif
2382 
2383  while(*sampling_local)
2384  {
2385 
2386  //Sampling power
2387  return_code = nvmlDeviceGetPowerUsage( device_local, power_local );
2388  if ( return_code != NVML_SUCCESS) {
2389  continue;
2390  }
2391  //Sampling time, do it after sampling power,
2392  //because sampling power is costly,
2393  //time stamp is taken long time ago
2394  //after a power sample is obtained,
2395  //may give the false feeling that sampling stops ealier
2396  //than the release of stop signal
2397  //clock_gettime(CLOCK_MONOTONIC, time_local);
2398  METERPU_TIME_MEASURE(time_local);
2399 
2400 #if METERPU_VERBOSE >= 4
2401  PRINT_CLASS_FUNC_NAME_CONT<<"current power("<<*power_local<<")\n";
2402 #endif
2403 
2404  //Add time and power pair to database
2405  //lu/ it is a deep copy, thus safe!
2406  power_db_local->push_back(*power_local);
2407  time_db_local->push_back(*time_local);
2408  }
2409 
2410 #if METERPU_VERBOSE >= 2
2411  NVML_Energy<>::Time_DB_Type::iterator it=time_db_local->end();
2412  PRINT_CLASS_FUNC_NAME_CONT<<"Obtained stop signal, the last time stamp of sampling is: "
2413  <<*--it
2414  <<std::endl;
2415 #endif
2416 
2417  pthread_exit((void*) arg);
2418  }
2419  pthread_t thread;
2420  pthread_attr_t attr;
2421  //void persist_DB() const{
2422  //#if METERPU_VERBOSE >= 1
2423  //PRINT_CLASS_FUNC_NAME("called");
2424  //#endif
2425 
2429  //std::ofstream out1("appgen_power.csv");
2430  //if(!out1){
2431  //std::cout<<"File creation error!"<<std::endl;
2432  //exit(1);
2433  //}
2434  //out1<<"\"tvsec\",\"tvnsec\",\"power\""<<std::endl;
2435  //for(unsigned int i=0;i<time_db.size();++i)
2436  //out1<<time_db[i].tv_sec<<","<<time_db[i].tv_nsec<<","<<power_db[i]<<std::endl;
2437 
2438  //}
2439  public:
2440  void start()
2441  {
2442  int rc;
2443 
2444  pthread_attr_init(&attr);
2445  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
2446 
2447  //Set the thread runnable flag
2448  sampling=true;
2449 
2450 #if METERPU_VERBOSE >= 2
2451  PRINT_CLASS_FUNC_NAME_CONT<<"sampling("<<sampling<<")\n";
2452 #endif
2453 
2454  //Release the thread
2455  rc = pthread_create(&thread, &attr, thread_program, (void *)this);
2456 #if METERPU_VERBOSE >= 1
2457  PRINT_CLASS_FUNC_NAME("thread released");
2458 #endif
2459  if (rc) {
2460  printf("ERROR; return code from pthread_create() is %d\n", rc);
2461  exit(-1);
2462  }
2463  }
2464  void stop(){
2465  int rc;
2466  void *status;
2467 
2468  //Set stop signal
2469  sampling=false;
2470 
2471  //Wait for thread to join
2472  rc = pthread_join(thread, &status);
2473  if (rc) {
2474  printf("ERROR; return code from pthread_join() is %d\n", rc);
2475  exit(-1);
2476  }
2477  pthread_attr_destroy(&attr);
2478 
2479 #if METERPU_VERBOSE >= 1
2480 
2481  NVML_Energy<>::Time_DB_Type::iterator it=time_db.end();
2482  CPU_Time_Measurement_Controller temp(*time_db.begin(),*--it);
2483  temp.calc();
2484  PRINT_CLASS_FUNC_NAME_CONT<<"time period of sampling is: "
2485  <<temp.get_value()
2486  <<std::endl;
2487  PRINT_CLASS_FUNC_NAME_CONT<<"number of sampling is: "
2488  <<power_db.size()
2489  <<std::endl;
2490  PRINT_CLASS_FUNC_NAME_CONT<<"The first sample of power is: "
2491  <<*power_db.begin()
2492  <<std::endl;
2493  PRINT_CLASS_FUNC_NAME_CONT<<"sampling rate is: "
2494  <<power_db.size()/temp.get_value()
2495  <<" #samples/sec."
2496  <<std::endl;
2497 #endif
2498 
2499 
2500  }
2501  void calc(){
2502 #if METERPU_VERBOSE >= 1
2503  PRINT_CLASS_FUNC_NAME("called");
2504 #endif
2505  //persist_DB();
2506 
2507  //reset state for next measurement
2508  reset_state();
2509 
2510  }
2511  void show_meter_reading() const {PRINT_FUNC_NAME_CONT<<"Not applicable to call me..."<<std::endl;}
2512  void set_device(const nvmlDevice_t &device_handle){device=device_handle;}
2513  const NVML_Energy<>::Power_Unit &get_power() const {return power;}
2514  const NVML_Energy<>::Power_DB_Type &get_power_db() const {return power_db;}
2515  const NVML_Energy<>::Time_DB_Type &get_time_db() const {return time_db;}
2516  NVML_Energy<>::Time_Unit const &get_stop_time(){
2517  assert(time_db.size()>=1);
2518  //TODO: use back() method here
2519  NVML_Energy<>::Time_DB_Type::iterator it=time_db.end();
2520  return *--it;
2521  }
2522  NVML_Energy<>::Time_Unit const &get_start_time(){
2523  assert(time_db.size()>=1);
2524  //NVML_Energy<>::Time_DB_Type::iterator it=time_db.begin();
2525  //return *it;
2526  //TODO: use front() method here
2527  return *time_db.begin();
2528  }
2529  void reset_state(){time_db.clear();power_db.clear();}
2530 
2531  /*}}}*/
2532  };
2533  Sampling_Thread_Controller sampling_thread_controller;
2534  /*}}}*/
2535  };
2536 #endif
2537 
2538 
2539 #ifdef ENABLE_SYSTEM_ENERGY
2540 
2541  //Micro for manually loop unrolling
2542  /*{{{*/
2543 
2550 #define POINTER ((Meter<NVML_Energy<first_device_id> >*)x[array_index])
2551 
2552 #define NOTHING
2553 
2560 #define LOOPER(name, return_type, base_code, general_code) \
2561  template <GPU_Device_Id_Type ...default_case> \
2562  struct name { static return_type apply(void *x[]) { base_code } }; \
2563  template <GPU_Device_Id_Type array_index, \
2564  GPU_Device_Id_Type first_device_id, \
2565  GPU_Device_Id_Type ...rest> \
2566  struct name <array_index,first_device_id, rest...> \
2567  { \
2568  static return_type apply(void *x[]) \
2569  { \
2570  general_code \
2571  name <array_index+1,rest...>::apply(x); \
2572  } \
2573  };
2574  /*}}}*/
2575 
2577 
2581  template<GPU_Device_Id_Type ...gpu_device_ids>
2582  struct System_Energy_Measurement_Controller : public Measurement_Controller
2583  {
2584  /*{{{*/
2585  private:
2586  LOOPER(GPU_Meter_Constructor, void, NOTHING, x[array_index]=new Meter<NVML_Energy<first_device_id> >;)
2587  LOOPER(GPU_Meter_Destructor, void, NOTHING, delete POINTER;)
2588  LOOPER(GPU_Meter_Start, void, NOTHING, POINTER->start();)
2589  LOOPER(GPU_Meter_Stop, void, NOTHING, POINTER->stop();)
2590  LOOPER(GPU_Meter_Cal, void, NOTHING, POINTER->calc();)
2591  LOOPER(GPU_Meter_Get_Meter_Reading, NVML_Energy<>::ResultType, return 0;, return POINTER->get_value()+)
2592 
2593  private:
2595  enum{N=sizeof...(gpu_device_ids)};
2596  void* gpu_meters[N];
2598  public:
2600  GPU_Meter_Constructor<0,gpu_device_ids...>::apply(gpu_meters);
2601  }
2603  GPU_Meter_Destructor<0,gpu_device_ids...>::apply(gpu_meters);
2604  }
2605  void init(){}
2606  void start(){
2607  GPU_Meter_Start<0,gpu_device_ids...>::apply(gpu_meters);
2608  cpu_meter.start();
2609  }
2610  void stop(){
2611  cpu_meter.stop();
2612  GPU_Meter_Stop<0,gpu_device_ids...>::apply(gpu_meters);
2613  }
2614  void calc(){
2615  meter_reading=0;
2616  cpu_meter.calc();
2617  meter_reading+=cpu_meter.get_value();
2618  GPU_Meter_Cal<0,gpu_device_ids...>::apply(gpu_meters);
2619  meter_reading=cpu_meter.get_value()
2620  + GPU_Meter_Get_Meter_Reading<0,gpu_device_ids...>::apply(gpu_meters)
2621  ;
2622  }
2630  void show_meter_reading() const {std::cout<<"[System Energy Meter] Energy consumed is: "<<get_value()<<" milli Joules."<<std::endl;}
2631  /*}}}*/
2632  };
2633 #endif
2634 
2635  // end of group3
2637  /*}}}*/
2638 
2639 
2640 }
2641 /*}}}*/
2642 
2643 
2644 #endif
void stop()
mark the end of a measurement phase/period.
Definition: MeterPU.h:914
PCM_Energy::DRAM_Energy_Type dram_energy
Definition: MeterPU.h:945
const CPU_Time::Time_Unit & get_start_time() const
Definition: MeterPU.h:1060
void init()
Definition: MeterPU.h:605
PCM_Energy::ResultType const & get_dram_energy() const
Definition: MeterPU.h:941
std::string path
Definition: MeterPU.h:1433
const NVML_Energy::Power_Unit & get_power() const
Definition: MeterPU.h:1832
GPU_Device_Id_Type device_id
Device id.
Definition: MeterPU.h:1012
void * gpu_meters[N]
Definition: MeterPU.h:2596
void dumpOriginalPowerData() const
Definition: MeterPU.h:1467
Definition: MeterPU.h:511
void dumpTimeEvent()
Definition: MeterPU.h:1442
void stop()
mark the end of a measurement phase/period.
Definition: MeterPU.h:1784
nvmlDevice_t device
Device handle.
Definition: MeterPU.h:1017
void start()
mark the start of a measurement phase/period.
Definition: MeterPU.h:875
CPU_Time::ResultType meter_reading
Definition: MeterPU.h:799
CPU_Time::ResultType diff()
Definition: MeterPU.h:1068
NVML_Energy::Hp_Power_DB_Type correctedPowerDB
Definition: MeterPU.h:1536
DECLARE_CLASS_NAME("CPU_Time_Environment_Init")
Meter_Traits< Type >::ResultType const & get_value() const
Get calculated metric value, require calc() to be called already.
Definition: MeterPU.h:251
nvmlDevice_t device
Device handle.
Definition: MeterPU.h:1695
Cuda Timer Traits.
Definition: MeterPU.h:404
void start()
mark the start of a measurement phase/period.
Definition: MeterPU.h:807
#define PRINT_FUNC_NAME_CONT
Definition: MeterPU.h:88
std::vector< Power_Unit > Power_DB_Type
Definition: MeterPU.h:456
nvmlDevice_t device
Definition: MeterPU.h:1056
PCM_Energy::ResultType const & get_cpu_energy() const
Definition: MeterPU.h:940
NVML_Energy::Power_Unit power
Power value for each sample.
Definition: MeterPU.h:1700
NVML_Energy::Time_Unit const & get_start_time()
Definition: MeterPU.h:1844
DECLARE_CLASS_NAME("CUDA_Time_Environment_Init")
double ResultType
Definition: MeterPU.h:420
void show_meter_reading() const
Print the calculated metric value to standard output, requires an invocation of calc() already done...
Definition: MeterPU.h:2630
long double Hp_Power_Unit
High precision power unit.
Definition: MeterPU.h:453
Sampling Thread Controller.
Definition: MeterPU.h:1681
SystemCounterState after_sstate
Definition: MeterPU.h:965
cudaEvent_t start_time
Definition: MeterPU.h:855
void init()
Definition: MeterPU.h:2605
#define PRINT_CLASS_FUNC_NAME_CONT
Definition: MeterPU.h:92
CPU and DRAME Energy Measurement Controller.
Definition: MeterPU.h:901
std::vector< Hp_Power_Unit > Hp_Power_DB_Type
Definition: MeterPU.h:457
Type::Measurement_Controller Measurement_Controller
Measurement Controller Object.
Definition: MeterPU.h:319
PCM Energy Traits.
Definition: MeterPU.h:417
CUDA-enabled GPU Time Measurement Controller.
Definition: MeterPU.h:851
void init()
Definition: MeterPU.h:584
void show_meter_reading() const
Print the calculated metric value to standard output, requires an invocation of calc() already done...
Definition: MeterPU.h:839
CPU_Time::Time_Unit start_time
Definition: MeterPU.h:1058
virtual void calc()=0
calculate the metric value between start() and stop().
CPU_Time::Time_Unit stop_time
Definition: MeterPU.h:798
CPU_Time::Time_Unit start_time
Definition: MeterPU.h:797
NVML_Energy::Time_DB_Type Data_Type
Definition: MeterPU.h:521
virtual void start()=0
mark the start of a measurement phase/period.
CPU_Time::ResultType operator-(CPU_Time::Time_Unit const &stop_time, CPU_Time::Time_Unit const &start_time)
Calculate elapsed time between two time stamp.
Definition: MeterPU.h:347
void init()
Definition: MeterPU.h:726
NVML_Energy::Energy_Unit calAreaForOneTrapezoid(NVML_Energy<>::Time_Unit start_time, NVML_Energy<>::Time_Unit end_time, NVML_Energy<>::Hp_Power_Unit start_power, NVML_Energy<>::Hp_Power_Unit end_power)
Definition: MeterPU.h:1657
void stop()
mark the end of a measurement phase/period.
Definition: MeterPU.h:1085
PCM_Energy_Measurement_Controller Measurement_Controller
Definition: MeterPU.h:421
void start()
mark the start of a measurement phase/period.
Definition: MeterPU.h:1760
#define PRINT_CLASS_FUNC_NAME(message)
Definition: MeterPU.h:91
DECLARE_CLASS_NAME("PCM_Energy_Environment_Init")
CUDA_Time::ResultType const & get_value() const
Get calculated metric value, require calc() to be called already.
Definition: MeterPU.h:890
DECLARE_CLASS_NAME("NVML_Energy_Measurement_Controller")
NVML_Energy::Power_DB_Type Data_Type
Definition: MeterPU.h:507
DECLARE_CLASS_NAME("CPU_Time_Measurement_Controller")
struct timespec Time_Unit
Time stamp unit.
Definition: MeterPU.h:337
void show_meter_reading()
Write measurement value on standard output with its unit.
Definition: MeterPU.h:258
NVML_Energy::Energy_Unit get_value() const
Get calculated metric value, require calc() to be called already.
Definition: MeterPU.h:1173
void calc()
calculate the metric value between start() and stop().
Definition: MeterPU.h:1103
PCM_Energy::ResultType meter_reading
Definition: MeterPU.h:943
NVML_Energy::Power_DB_Const_Iterator_Type Const_Iterator_Type
Definition: MeterPU.h:508
double CPU_Energy_Type
Definition: MeterPU.h:423
NVML_Energy::Hp_Power_DB_Type Data_Type
Definition: MeterPU.h:514
CPU_Time_Measurement_Controller Measurement_Controller
Definition: MeterPU.h:331
~PCM_Energy_Measurement_Controller()
Definition: MeterPU.h:957
void set_device(const nvmlDevice_t &device_handle)
Definition: MeterPU.h:1831
unsigned int Power_Unit
Definition: MeterPU.h:448
double ResultType
Definition: MeterPU.h:330
CUDA_Time_Measurement_Controller Measurement_Controller
Definition: MeterPU.h:408
void start()
mark the start of a measurement phase/period.
Definition: MeterPU.h:908
void init()
Definition: MeterPU.h:869
void correctPowerSamplesByBurtscherApproach()
Definition: MeterPU.h:1537
#define METERPU_TIME_MEASURE(x)
Definition: MeterPU.h:126
void calc()
calculate the metric value between start() and stop().
Definition: MeterPU.h:877
System_Energy_Measurement_Controller< gpu_ids...> Measurement_Controller
Definition: MeterPU.h:474
CUDA_Time_Measurement_Controller()
Definition: MeterPU.h:859
NVML_Energy_Environment_Init Environment_Init_Type
Definition: MeterPU.h:442
Power_DB_Type::const_iterator Power_DB_Const_Iterator_Type
Definition: MeterPU.h:458
void calc()
calculate the metric value between start() and stop().
Definition: MeterPU.h:920
static std::string header_message()
Definition: MeterPU.h:490
#define POINTER
A macro for downcast a void pointer to a NVML Energy Meter.
Definition: MeterPU.h:2550
DECLARE_CLASS_NAME("PCM_Energy_Measurement_Controller")
Time_DB_Type::const_iterator Time_DB_Const_Iterator_Type
Definition: MeterPU.h:455
void init()
Definition: MeterPU.h:623
DECLARE_CLASS_NAME("PCM_Energy_Environment_Init")
static std::string header_message()
Definition: MeterPU.h:520
void init()
Definition: MeterPU.h:801
bool bash_exe(const std::string &cmd, std::vector< std::string > &out)
Definition: MeterPU.h:132
static void * thread_program(void *arg)
Definition: MeterPU.h:1711
Time Traits.
Definition: MeterPU.h:327
void set_power_db(const NVML_Energy<>::Power_DB_Type &x)
Definition: MeterPU.h:1838
Meter< PCM_Energy > cpu_meter
Definition: MeterPU.h:2594
CUDA_Time::ResultType meter_reading
Definition: MeterPU.h:857
NVML_Energy_Device_Init nvml_energy_device_init
Definition: MeterPU.h:1055
unsigned int GPU_Device_Id_Type
Definition: MeterPU.h:175
void set_time_db(const NVML_Energy<>::Time_DB_Type &x)
Definition: MeterPU.h:1837
NVML_Energy::Power_DB_Type & get_power_db_nonconst()
Definition: MeterPU.h:1835
cudaEvent_t stop_time
Definition: MeterPU.h:856
const CPU_Time::Time_Unit & get_stop_time() const
Definition: MeterPU.h:1061
Definition: MeterPU.h:518
Traits Interface.
Definition: MeterPU.h:165
CPU_Time_Measurement_Controller(CPU_Time::Time_Unit const &start_time_p, CPU_Time::Time_Unit const &stop_time_p)
Definition: MeterPU.h:792
NVML_Energy::Power_DB_Type power_db
Definition: MeterPU.h:1710
Type::Const_Iterator_Type Const_Iterator_Type
Definition: MeterPU.h:492
bool operator==(CPU_Time::Time_Unit const &small_time, CPU_Time::Time_Unit const &large_time)
Check if two time stamps are the same.
Definition: MeterPU.h:379
Sampling_Thread_Controller sampling_thread_controller
Definition: MeterPU.h:1852
void start()
mark the start of a measurement phase/period.
Definition: MeterPU.h:2606
System_Energy_Environment_Init Environment_Init_Type
Definition: MeterPU.h:473
CPU_Time::ResultType const & get_value() const
Get calculated metric value, require calc() to be called already.
Definition: MeterPU.h:835
void init()
Initialize a GPU device.
Definition: MeterPU.h:994
void show_meter_reading() const
Print the calculated metric value to standard output, requires an invocation of calc() already done...
Definition: MeterPU.h:1174
void init()
Definition: MeterPU.h:907
Meter_Traits< Type >::ResultType meter_reading
The variable used to store the calculated.
Definition: MeterPU.h:280
PCM_Energy::CPU_Energy_Type cpu_energy
Definition: MeterPU.h:944
CPU_Time_Environment_Init Environment_Init_Type
Definition: MeterPU.h:329
System_Energy_Measurement_Controller()
Definition: MeterPU.h:2599
PCM_Energy_Environment_Init Environment_Init_Type
Definition: MeterPU.h:419
Definition: MeterPU.h:488
bool operator<=(CPU_Time::Time_Unit const &small_time, CPU_Time::Time_Unit const &large_time)
Check if a time stamp is earlier or equal to another.
Definition: MeterPU.h:392
CPU_Time_Measurement_Controller()
Definition: MeterPU.h:791
Library Initializer Interface.
Definition: MeterPU.h:550
void calc()
calculate measurement value
Definition: MeterPU.h:240
void init()
Definition: MeterPU.h:565
void show_meter_reading() const
Print the calculated metric value to standard output, requires an invocation of calc() already done...
Definition: MeterPU.h:1830
const NVML_Energy::Power_DB_Type & get_power_db() const
Definition: MeterPU.h:1833
void fillPowerValuesAtStartAndEnd()
Definition: MeterPU.h:1592
NVML_Energy::Energy_Unit meter_reading
Definition: MeterPU.h:1057
Meter_Traits< Type >::Environment_Init_Type environment_init_object
Native library initializer.
Definition: MeterPU.h:268
GPU Energy Traits.
Definition: MeterPU.h:182
GPU Time Library Initializer.
Definition: MeterPU.h:578
void dumpCorrectedPowerData()
Definition: MeterPU.h:1470
Energy_Unit ResultType
Definition: MeterPU.h:443
void update_cpu_energy()
Definition: MeterPU.h:946
NVML_Energy::ResultType meter_reading
Definition: MeterPU.h:2597
void removeRedundantSamplesByDistance(const CPU_Time::ResultType &time_distance_ms)
Definition: MeterPU.h:1473
void update_dram_energy()
Definition: MeterPU.h:947
NVML_Energy::Energy_Unit calTotalArea(NVML_Energy<>::Hp_Power_DB_Type const &power_db, NVML_Energy<>::Time_DB_Type const &time_db)
Definition: MeterPU.h:1633
Type::Environment_Init_Type Environment_Init_Type
Object for initialization of a native measurement library.
Definition: MeterPU.h:309
void show_meter_reading() const
Print the calculated metric value to standard output, requires an invocation of calc() already done...
Definition: MeterPU.h:937
NVML_Energy::Hp_Power_DB_Const_Iterator_Type Const_Iterator_Type
Definition: MeterPU.h:515
~CUDA_Time_Measurement_Controller()
Definition: MeterPU.h:864
Type::ResultType ResultType
Result type.
Definition: MeterPU.h:314
void calc()
calculate the metric value between start() and stop().
Definition: MeterPU.h:1821
#define LOOPER(name, return_type, base_code, general_code)
A macro to build variadic template to recursively apply code snippets.
Definition: MeterPU.h:2560
Meter_Traits< Type >::Measurement_Controller measurement_controller_object
Measurement controller.
Definition: MeterPU.h:274
NVML_Energy::Energy_Unit calTotalArea()
Definition: MeterPU.h:1624
static std::string header_message()
Definition: MeterPU.h:506
CPU Time Library Initializer.
Definition: MeterPU.h:559
const NVML_Energy::Time_DB_Type & get_time_db() const
Definition: MeterPU.h:1834
System_Energy::ResultType const & get_value() const
Get calculated metric value, require calc() to be called already.
Definition: MeterPU.h:2629
virtual void init()=0
Meter()
Definition: MeterPU.h:224
CPU Time Measurement Controller.
Definition: MeterPU.h:787
void stop()
mark the end of a measurement phase/period.
Definition: MeterPU.h:815
float ResultType
Definition: MeterPU.h:407
System Energy Measurement Controller.
Definition: MeterPU.h:198
PCM * pcm
Definition: MeterPU.h:963
CUDA_Time_Environment_Init Environment_Init_Type
Definition: MeterPU.h:406
virtual void stop()=0
mark the end of a measurement phase/period.
void calc()
calculate the metric value between start() and stop().
Definition: MeterPU.h:2614
double ResultType
Definition: MeterPU.h:475
void start()
start a measurement
Definition: MeterPU.h:232
SystemCounterState before_sstate
Definition: MeterPU.h:964
void record_start_time()
Definition: MeterPU.h:1062
void calc()
calculate the metric value between start() and stop().
Definition: MeterPU.h:822
CPU_Time::Time_Unit stop_time
Definition: MeterPU.h:1059
GPU Energy Library Initializer.
Definition: MeterPU.h:619
NVML_Energy::Time_Unit time
Time value for each sample.
Definition: MeterPU.h:1704
double DRAM_Energy_Type
Definition: MeterPU.h:424
#define NOTHING
Definition: MeterPU.h:2552
CUDA-enabled GPU Energy Measurement Controller.
Definition: MeterPU.h:184
System Energy Library Initializer.
Definition: MeterPU.h:720
void stop()
mark the end of a measurement phase/period.
Definition: MeterPU.h:876
Measurement Controller Interface.
Definition: MeterPU.h:754
NVML_Energy::Time_DB_Type & get_time_db_nonconst()
Definition: MeterPU.h:1836
PCM_Energy_Measurement_Controller()
Definition: MeterPU.h:949
Hp_Power_DB_Type::const_iterator Hp_Power_DB_Const_Iterator_Type
Definition: MeterPU.h:459
static std::string header_message()
Definition: MeterPU.h:513
The software multi-meters.
Definition: MeterPU.h:221
NVML library init and teardown.
Definition: MeterPU.h:630
Type::Data_Type Data_Type
Definition: MeterPU.h:491
void stop()
mark the end of a measurement phase/period.
Definition: MeterPU.h:2610
virtual void show_meter_reading() const =0
Print the calculated metric value to standard output, requires an invocation of calc() already done...
double Energy_Unit
Definition: MeterPU.h:440
NVML_Energy::Time_Unit const & get_stop_time()
Definition: MeterPU.h:1839
void set_device_id(GPU_Device_Id_Type id)
Definition: MeterPU.h:1026
void show_meter_reading() const
Print the calculated metric value to standard output, requires an invocation of calc() already done...
Definition: MeterPU.h:881
static NVML_Energy::Power_Unit get_UNREALISTIC_POWER_VALUE()
Definition: MeterPU.h:1689
void resetDir()
Definition: MeterPU.h:1434
NVML_Energy_Measurement_Controller< device_id > Measurement_Controller
Definition: MeterPU.h:444
struct timespec Time_Unit
Definition: MeterPU.h:447
~System_Energy_Measurement_Controller()
Definition: MeterPU.h:2602
CPU Energy Library Initializer.
Definition: MeterPU.h:599
void dumpTwoVectors(const std::string &filename, const std::string &header, const T1 &keys, const T2 &values) const
Definition: MeterPU.h:1450
bool operator<(CPU_Time::Time_Unit const &small_time, CPU_Time::Time_Unit const &large_time)
Check if a time stamp is earlier than another.
Definition: MeterPU.h:364
void record_stop_time()
Definition: MeterPU.h:1065
NVML_Energy::Time_DB_Const_Iterator_Type Const_Iterator_Type
Definition: MeterPU.h:522
static void print(const Data_Type &a)
Definition: MeterPU.h:493
PCM_Energy::ResultType const & get_value() const
Get calculated metric value, require calc() to be called already.
Definition: MeterPU.h:936
#define DECLARE_CLASS_NAME(name)
Definition: MeterPU.h:90
Definition: MeterPU.h:504
NVML_Energy::Time_DB_Type time_db
Power data database.
Definition: MeterPU.h:1709
static void init_NVML()
Definition: MeterPU.h:639
std::vector< Time_Unit > Time_DB_Type
Definition: MeterPU.h:454
const nvmlDevice_t & get_device() const
Definition: MeterPU.h:1052
void start()
mark the start of a measurement phase/period.
Definition: MeterPU.h:1071
System Energy Traits.
Definition: MeterPU.h:196
void stop()
stop a measurement
Definition: MeterPU.h:236