00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028 #ifdef __GNUC__
00029 #pragma interface
00030 #endif
00031
00032 #ifndef _util_group_memory_h
00033 #define _util_group_memory_h
00034
00035 #include <iostream>
00036
00037 #include <scconfig.h>
00038 #include <util/class/class.h>
00039 #include <util/group/thread.h>
00040
00041 namespace sc {
00042
00043 #if 0 // this can be used to catch accidental conversions to int
00044 class distsize_t {
00045 friend size_t distsize_to_size(const distsize_t &a);
00046 friend distsize_t operator *(const int &a,const distsize_t &b);
00047 friend distsize_t operator +(const int &a,const distsize_t &b);
00048 friend distsize_t operator -(const int &a,const distsize_t &b);
00049 friend distsize_t operator /(const int &a,const distsize_t &b);
00050 friend distsize_t operator %(const int &a,const distsize_t &b);
00051 friend ostream& operator <<(ostream& o, const distsize_t &s);
00052 private:
00053 unsigned long long s;
00054 public:
00055 distsize_t(): s(999999999999999LL) {}
00056 distsize_t(int a): s(a) {}
00057 distsize_t(unsigned int a): s(a) {}
00058 distsize_t(unsigned long long a): s(a) {}
00059 distsize_t &operator =(const distsize_t &a)
00060 { s=a.s; return *this; }
00061 distsize_t &operator +=(const distsize_t &a)
00062 { s+=a.s; return *this; }
00063 distsize_t operator *(const distsize_t &a) const
00064 { return s*a.s; }
00065 distsize_t operator +(const distsize_t &a) const
00066 { return s+a.s; }
00067 distsize_t operator -(const distsize_t &a) const
00068 { return s-a.s; }
00069 distsize_t operator /(const distsize_t &a) const
00070 { return s/a.s; }
00071 distsize_t operator %(const distsize_t &a) const
00072 { return s%a.s; }
00073 bool operator <(const distsize_t &a) const
00074 { return s<a.s; }
00075 bool operator <=(const distsize_t &a) const
00076 { return s<=a.s; }
00077 bool operator >(const distsize_t &a) const
00078 { return s>a.s; }
00079 bool operator >=(const distsize_t &a) const
00080 { return s>=a.s; }
00081 bool operator ==(const distsize_t &a) const
00082 { return s==a.s; }
00083 distsize_t operator *(const int &a) const
00084 { return s*a; }
00085 distsize_t operator +(const int &a) const
00086 { return s+a; }
00087 distsize_t operator -(const int &a) const
00088 { return s-a; }
00089 distsize_t operator /(const int &a) const
00090 { return s/a; }
00091 distsize_t operator %(const int &a) const
00092 { return s%a; }
00093 };
00094 inline distsize_t operator *(const int &a,const distsize_t &b)
00095 { return a*b.s; }
00096 inline distsize_t operator +(const int &a,const distsize_t &b)
00097 { return a+b.s; }
00098 inline distsize_t operator -(const int &a,const distsize_t &b)
00099 { return a-b.s; }
00100 inline distsize_t operator /(const int &a,const distsize_t &b)
00101 { return a/b.s; }
00102 inline distsize_t operator %(const int &a,const distsize_t &b)
00103 { return a%b.s; }
00104 inline ostream& operator <<(ostream& o, const distsize_t &s) { return o<<s.s; }
00105 inline size_t distsize_to_size(const distsize_t &a) {return a.s;}
00106 #elif defined(HAVE_LONG_LONG)
00107 typedef unsigned long long distsize_t;
00108 typedef long long distssize_t;
00109 inline size_t distsize_to_size(const distsize_t &a) {return a;}
00110 #else
00111 typedef unsigned long distsize_t;
00112 typedef long distssize_t;
00113 inline size_t distsize_to_size(const distsize_t &a) {return a;}
00114 #endif
00115
00124 class MemoryGrp: public DescribedClass {
00125 private:
00126 Ref<ThreadLock> *locks_;
00127 int nlock_;
00128
00129 void init_locks();
00130
00131
00132 protected:
00133
00134
00135 int me_;
00136 int n_;
00137 distsize_t *offsets_;
00138
00139
00140 int debug_;
00141
00142 void obtain_local_lock(size_t start, size_t fence);
00143 void release_local_lock(size_t start, size_t fence);
00144 public:
00145 MemoryGrp();
00146 MemoryGrp(const Ref<KeyVal>&);
00147 virtual ~MemoryGrp();
00148
00150 int me() const { return me_; }
00152 int n() const { return n_; }
00153
00157 virtual void set_localsize(size_t) = 0;
00159 size_t localsize() { return distsize_to_size(offsets_[me_+1]-offsets_[me_]); }
00161 virtual void *localdata() = 0;
00163 distsize_t localoffset() { return offsets_[me_]; }
00165 int size(int node)
00166 { return distsize_to_size(offsets_[node+1] - offsets_[node]); }
00168 distsize_t offset(int node) { return offsets_[node]; }
00170 distsize_t totalsize() { return offsets_[n_]; }
00171
00173 virtual void activate();
00175 virtual void deactivate();
00176
00178 virtual void *obtain_writeonly(distsize_t offset, int size) = 0;
00184 virtual void *obtain_readwrite(distsize_t offset, int size) = 0;
00186 virtual void *obtain_readonly(distsize_t offset, int size) = 0;
00188 virtual void release_readonly(void *data, distsize_t offset, int size) = 0;
00190 virtual void release_writeonly(void *data, distsize_t offset, int size)=0;
00193 virtual void release_readwrite(void *data, distsize_t offset, int size)=0;
00194
00195 virtual void sum_reduction(double *data, distsize_t doffset, int dsize);
00196 virtual void sum_reduction_on_node(double *data, size_t doffset, int dsize,
00197 int node = -1);
00198
00202 virtual void sync() = 0;
00203
00208 virtual void* malloc_local(size_t nbyte);
00209 virtual double* malloc_local_double(size_t ndouble);
00210
00212 virtual void free_local(void *data);
00213 virtual void free_local_double(double *data);
00214
00221 virtual void catchup();
00222
00224 virtual void print(std::ostream &o = ExEnv::out0()) const;
00225
00233 static MemoryGrp* initial_memorygrp(int &argc, char** argv);
00234 static MemoryGrp* initial_memorygrp();
00237 static void set_default_memorygrp(const Ref<MemoryGrp>&);
00243 static MemoryGrp* get_default_memorygrp();
00244 };
00245
00246
00252 template <class data_t>
00253 class MemoryGrpBuf {
00254 Ref<MemoryGrp> grp_;
00255 enum AccessType { None, Read, Write, ReadWrite };
00256 AccessType accesstype_;
00257 data_t *data_;
00258 distsize_t offset_;
00259 int length_;
00260 public:
00264 MemoryGrpBuf(const Ref<MemoryGrp> &);
00269 data_t *writeonly(distsize_t offset, int length);
00274 data_t *readwrite(distsize_t offset, int length);
00279 const data_t *readonly(distsize_t offset, int length);
00283 data_t *writeonly_on_node(size_t offset, int length, int node = -1);
00284 data_t *readwrite_on_node(size_t offset, int length, int node = -1);
00285 const data_t *readonly_on_node(size_t offset, int length, int node = -1);
00289 void release();
00291 int length() const { return length_; }
00292 };
00293
00295
00296
00297 template <class data_t>
00298 MemoryGrpBuf<data_t>::MemoryGrpBuf(const Ref<MemoryGrp> & grp)
00299 {
00300 grp_ = grp;
00301 accesstype_ = None;
00302 }
00303
00304 template <class data_t>
00305 data_t *
00306 MemoryGrpBuf<data_t>::writeonly(distsize_t offset, int length)
00307 {
00308 if (accesstype_ != None) release();
00309 data_ = (data_t *) grp_->obtain_writeonly(sizeof(data_t)*offset,
00310 sizeof(data_t)*length);
00311 offset_ = offset;
00312 length_ = length;
00313 accesstype_ = Write;
00314 return data_;
00315 }
00316
00317 template <class data_t>
00318 data_t *
00319 MemoryGrpBuf<data_t>::readwrite(distsize_t offset, int length)
00320 {
00321 if (accesstype_ != None) release();
00322 data_ = (data_t *) grp_->obtain_readwrite(sizeof(data_t)*offset,
00323 sizeof(data_t)*length);
00324 offset_ = offset;
00325 length_ = length;
00326 accesstype_ = ReadWrite;
00327 return data_;
00328 }
00329
00330 template <class data_t>
00331 const data_t *
00332 MemoryGrpBuf<data_t>::readonly(distsize_t offset, int length)
00333 {
00334 if (accesstype_ != None) release();
00335 data_ = (data_t *) grp_->obtain_readonly(sizeof(data_t)*offset,
00336 sizeof(data_t)*length);
00337 offset_ = offset;
00338 length_ = length;
00339 accesstype_ = Read;
00340 return data_;
00341 }
00342
00343 template <class data_t>
00344 data_t *
00345 MemoryGrpBuf<data_t>::writeonly_on_node(size_t offset, int length, int node)
00346 {
00347 if (node == -1) node = grp_->me();
00348 return writeonly(offset + grp_->offset(node)/sizeof(data_t), length);
00349 }
00350
00351 template <class data_t>
00352 data_t *
00353 MemoryGrpBuf<data_t>::readwrite_on_node(size_t offset, int length, int node)
00354 {
00355 if (node == -1) node = grp_->me();
00356 return readwrite(offset + grp_->offset(node)/sizeof(data_t), length);
00357 }
00358
00359 template <class data_t>
00360 const data_t *
00361 MemoryGrpBuf<data_t>::readonly_on_node(size_t offset, int length, int node)
00362 {
00363 if (node == -1) node = grp_->me();
00364 return readonly(offset + grp_->offset(node)/sizeof(data_t), length);
00365 }
00366
00367 template <class data_t>
00368 void
00369 MemoryGrpBuf<data_t>::release()
00370 {
00371 if (accesstype_ == Write)
00372 grp_->release_writeonly((data_t *)data_,
00373 sizeof(data_t)*offset_, sizeof(data_t)*length_);
00374 if (accesstype_ == Read)
00375 grp_->release_readonly(data_, sizeof(data_t)*offset_,
00376 sizeof(data_t)*length_);
00377 if (accesstype_ == ReadWrite)
00378 grp_->release_readwrite(data_, sizeof(data_t)*offset_,
00379 sizeof(data_t)*length_);
00380
00381 accesstype_ = None;
00382 }
00383
00384 }
00385
00386 #endif
00387
00388
00389
00390
00391