00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028 #ifdef __GNUC__
00029 #pragma interface
00030 #endif
00031
00032 #ifndef _util_group_memory_h
00033 #define _util_group_memory_h
00034
00035 #include <iostream>
00036
00037 #include <scconfig.h>
00038 #include <util/class/class.h>
00039 #include <util/group/thread.h>
00040
00041 namespace sc {
00042
00043 #if 0 // this can be used to catch accidental conversions to int
00044 class distsize_t {
00045 friend size_t distsize_to_size(const distsize_t &a);
00046 friend distsize_t operator *(const int &a,const distsize_t &b);
00047 friend distsize_t operator +(const int &a,const distsize_t &b);
00048 friend distsize_t operator -(const int &a,const distsize_t &b);
00049 friend distsize_t operator /(const int &a,const distsize_t &b);
00050 friend distsize_t operator %(const int &a,const distsize_t &b);
00051 friend ostream& operator <<(ostream& o, const distsize_t &s);
00052 private:
00053 unsigned long long s;
00054 public:
00055 distsize_t(): s(999999999999999LL) {}
00056 distsize_t(int a): s(a) {}
00057 distsize_t(unsigned int a): s(a) {}
00058 distsize_t(unsigned long long a): s(a) {}
00059 distsize_t &operator =(const distsize_t &a)
00060 { s=a.s; return *this; }
00061 distsize_t &operator +=(const distsize_t &a)
00062 { s+=a.s; return *this; }
00063 distsize_t operator *(const distsize_t &a) const
00064 { return s*a.s; }
00065 distsize_t operator +(const distsize_t &a) const
00066 { return s+a.s; }
00067 distsize_t operator -(const distsize_t &a) const
00068 { return s-a.s; }
00069 distsize_t operator /(const distsize_t &a) const
00070 { return s/a.s; }
00071 distsize_t operator %(const distsize_t &a) const
00072 { return s%a.s; }
00073 bool operator <(const distsize_t &a) const
00074 { return s<a.s; }
00075 bool operator <=(const distsize_t &a) const
00076 { return s<=a.s; }
00077 bool operator >(const distsize_t &a) const
00078 { return s>a.s; }
00079 bool operator >=(const distsize_t &a) const
00080 { return s>=a.s; }
00081 bool operator ==(const distsize_t &a) const
00082 { return s==a.s; }
00083 distsize_t operator *(const int &a) const
00084 { return s*a; }
00085 distsize_t operator +(const int &a) const
00086 { return s+a; }
00087 distsize_t operator -(const int &a) const
00088 { return s-a; }
00089 distsize_t operator /(const int &a) const
00090 { return s/a; }
00091 distsize_t operator %(const int &a) const
00092 { return s%a; }
00093 };
00094 inline distsize_t operator *(const int &a,const distsize_t &b)
00095 { return a*b.s; }
00096 inline distsize_t operator +(const int &a,const distsize_t &b)
00097 { return a+b.s; }
00098 inline distsize_t operator -(const int &a,const distsize_t &b)
00099 { return a-b.s; }
00100 inline distsize_t operator /(const int &a,const distsize_t &b)
00101 { return a/b.s; }
00102 inline distsize_t operator %(const int &a,const distsize_t &b)
00103 { return a%b.s; }
00104 inline ostream& operator <<(ostream& o, const distsize_t &s) { return o<<s.s; }
00105 inline size_t distsize_to_size(const distsize_t &a) {return a.s;}
00106 #elif defined(HAVE_LONG_LONG)
00107 typedef unsigned long long distsize_t;
00108 typedef long long distssize_t;
00109 inline size_t distsize_to_size(const distsize_t &a) {return a;}
00110 #else
00111 typedef unsigned long distsize_t;
00112 typedef long distssize_t;
00113 inline size_t distsize_to_size(const distsize_t &a) {return a;}
00114 #endif
00115
00155 class MemoryGrp: public DescribedClass {
00156 private:
00157 Ref<ThreadLock> *locks_;
00158 int nlock_;
00159
00160 void init_locks();
00161
00162
00163 protected:
00164
00165
00166 int me_;
00167 int n_;
00168 distsize_t *offsets_;
00169
00170
00171 int debug_;
00172
00173 void obtain_local_lock(size_t start, size_t fence);
00174 void release_local_lock(size_t start, size_t fence);
00175 public:
00176 MemoryGrp();
00177 MemoryGrp(const Ref<KeyVal>&);
00178 virtual ~MemoryGrp();
00179
00181 int me() const { return me_; }
00183 int n() const { return n_; }
00184
00188 virtual void set_localsize(size_t) = 0;
00190 size_t localsize() { return distsize_to_size(offsets_[me_+1]-offsets_[me_]); }
00192 virtual void *localdata() = 0;
00194 distsize_t localoffset() { return offsets_[me_]; }
00196 int size(int node)
00197 { return distsize_to_size(offsets_[node+1] - offsets_[node]); }
00199 distsize_t offset(int node) { return offsets_[node]; }
00201 distsize_t totalsize() { return offsets_[n_]; }
00202
00204 virtual void activate();
00206 virtual void deactivate();
00207
00209 virtual void *obtain_writeonly(distsize_t offset, int size) = 0;
00215 virtual void *obtain_readwrite(distsize_t offset, int size) = 0;
00217 virtual void *obtain_readonly(distsize_t offset, int size) = 0;
00219 virtual void release_readonly(void *data, distsize_t offset, int size) = 0;
00221 virtual void release_writeonly(void *data, distsize_t offset, int size)=0;
00224 virtual void release_readwrite(void *data, distsize_t offset, int size)=0;
00225
00226 virtual void sum_reduction(double *data, distsize_t doffset, int dsize);
00227 virtual void sum_reduction_on_node(double *data, size_t doffset, int dsize,
00228 int node = -1);
00229
00232 virtual void sync() = 0;
00233
00240 virtual void catchup();
00241
00243 virtual void print(std::ostream &o = ExEnv::out0()) const;
00244
00252 static MemoryGrp* initial_memorygrp(int &argc, char** argv);
00253 static MemoryGrp* initial_memorygrp();
00256 static void set_default_memorygrp(const Ref<MemoryGrp>&);
00258 static MemoryGrp* get_default_memorygrp();
00259 };
00260
00261
00267 template <class data_t>
00268 class MemoryGrpBuf {
00269 Ref<MemoryGrp> grp_;
00270 enum AccessType { None, Read, Write, ReadWrite };
00271 AccessType accesstype_;
00272 data_t *data_;
00273 distsize_t offset_;
00274 int length_;
00275 public:
00279 MemoryGrpBuf(const Ref<MemoryGrp> &);
00284 data_t *writeonly(distsize_t offset, int length);
00289 data_t *readwrite(distsize_t offset, int length);
00294 const data_t *readonly(distsize_t offset, int length);
00298 data_t *writeonly_on_node(size_t offset, int length, int node = -1);
00299 data_t *readwrite_on_node(size_t offset, int length, int node = -1);
00300 const data_t *readonly_on_node(size_t offset, int length, int node = -1);
00304 void release();
00306 int length() const { return length_; }
00307 };
00308
00310
00311
00312 template <class data_t>
00313 MemoryGrpBuf<data_t>::MemoryGrpBuf(const Ref<MemoryGrp> & grp)
00314 {
00315 grp_ = grp;
00316 accesstype_ = None;
00317 }
00318
00319 template <class data_t>
00320 data_t *
00321 MemoryGrpBuf<data_t>::writeonly(distsize_t offset, int length)
00322 {
00323 if (accesstype_ != None) release();
00324 data_ = (data_t *) grp_->obtain_writeonly(sizeof(data_t)*offset,
00325 sizeof(data_t)*length);
00326 offset_ = offset;
00327 length_ = length;
00328 accesstype_ = Write;
00329 return data_;
00330 }
00331
00332 template <class data_t>
00333 data_t *
00334 MemoryGrpBuf<data_t>::readwrite(distsize_t offset, int length)
00335 {
00336 if (accesstype_ != None) release();
00337 data_ = (data_t *) grp_->obtain_readwrite(sizeof(data_t)*offset,
00338 sizeof(data_t)*length);
00339 offset_ = offset;
00340 length_ = length;
00341 accesstype_ = ReadWrite;
00342 return data_;
00343 }
00344
00345 template <class data_t>
00346 const data_t *
00347 MemoryGrpBuf<data_t>::readonly(distsize_t offset, int length)
00348 {
00349 if (accesstype_ != None) release();
00350 data_ = (data_t *) grp_->obtain_readonly(sizeof(data_t)*offset,
00351 sizeof(data_t)*length);
00352 offset_ = offset;
00353 length_ = length;
00354 accesstype_ = Read;
00355 return data_;
00356 }
00357
00358 template <class data_t>
00359 data_t *
00360 MemoryGrpBuf<data_t>::writeonly_on_node(size_t offset, int length, int node)
00361 {
00362 if (node == -1) node = grp_->me();
00363 return writeonly(offset + grp_->offset(node)/sizeof(data_t), length);
00364 }
00365
00366 template <class data_t>
00367 data_t *
00368 MemoryGrpBuf<data_t>::readwrite_on_node(size_t offset, int length, int node)
00369 {
00370 if (node == -1) node = grp_->me();
00371 return readwrite(offset + grp_->offset(node)/sizeof(data_t), length);
00372 }
00373
00374 template <class data_t>
00375 const data_t *
00376 MemoryGrpBuf<data_t>::readonly_on_node(size_t offset, int length, int node)
00377 {
00378 if (node == -1) node = grp_->me();
00379 return readonly(offset + grp_->offset(node)/sizeof(data_t), length);
00380 }
00381
00382 template <class data_t>
00383 void
00384 MemoryGrpBuf<data_t>::release()
00385 {
00386 if (accesstype_ == Write)
00387 grp_->release_writeonly((data_t *)data_,
00388 sizeof(data_t)*offset_, sizeof(data_t)*length_);
00389 if (accesstype_ == Read)
00390 grp_->release_readonly(data_, sizeof(data_t)*offset_,
00391 sizeof(data_t)*length_);
00392 if (accesstype_ == ReadWrite)
00393 grp_->release_readwrite(data_, sizeof(data_t)*offset_,
00394 sizeof(data_t)*length_);
00395
00396 accesstype_ = None;
00397 }
00398
00399 }
00400
00401 #endif
00402
00403
00404
00405
00406