Package dbf :: Module tables
[hide private]

Source Code for Module dbf.tables

   1  "table definitions" 
   2  import os 
   3  import sys 
   4  import csv 
   5  import codecs 
   6  import locale 
   7  import unicodedata 
   8  import weakref 
   9  from array import array 
  10  from bisect import bisect_left, bisect_right 
  11  from decimal import Decimal 
  12  from shutil import copyfileobj 
  13  import dbf 
  14  from dbf import _io as io 
  15  from dbf.dates import Date, DateTime, Time 
  16  from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode, DoNotIndex 
  17   
  18  input_decoding = locale.getdefaultlocale()[1]    # treat non-unicode data as ... 
  19  default_codepage = 'cp1252'  # if no codepage specified on dbf creation, use this 
  20  return_ascii = False         # if True -- convert back to icky ascii, losing chars if no mapping 
  21  temp_dir = os.environ.get("DBF_TEMP") or os.environ.get("TEMP") or "" 
  22   
  23  version_map = { 
  24          '\x02' : 'FoxBASE', 
  25          '\x03' : 'dBase III Plus', 
  26          '\x04' : 'dBase IV', 
  27          '\x05' : 'dBase V', 
  28          '\x30' : 'Visual FoxPro', 
  29          '\x31' : 'Visual FoxPro (auto increment field)', 
  30          '\x43' : 'dBase IV SQL', 
  31          '\x7b' : 'dBase IV w/memos', 
  32          '\x83' : 'dBase III Plus w/memos', 
  33          '\x8b' : 'dBase IV w/memos', 
  34          '\x8e' : 'dBase IV w/SQL table', 
  35          '\xf5' : 'FoxPro w/memos'} 
  36   
  37  code_pages = { 
  38          '\x00' : ('ascii', "plain ol' ascii"), 
  39          '\x01' : ('cp437', 'U.S. MS-DOS'), 
  40          '\x02' : ('cp850', 'International MS-DOS'), 
  41          '\x03' : ('cp1252', 'Windows ANSI'), 
  42          '\x04' : ('mac_roman', 'Standard Macintosh'), 
  43          '\x08' : ('cp865', 'Danish OEM'), 
  44          '\x09' : ('cp437', 'Dutch OEM'), 
  45          '\x0A' : ('cp850', 'Dutch OEM (secondary)'), 
  46          '\x0B' : ('cp437', 'Finnish OEM'), 
  47          '\x0D' : ('cp437', 'French OEM'), 
  48          '\x0E' : ('cp850', 'French OEM (secondary)'), 
  49          '\x0F' : ('cp437', 'German OEM'), 
  50          '\x10' : ('cp850', 'German OEM (secondary)'), 
  51          '\x11' : ('cp437', 'Italian OEM'), 
  52          '\x12' : ('cp850', 'Italian OEM (secondary)'), 
  53          '\x13' : ('cp932', 'Japanese Shift-JIS'), 
  54          '\x14' : ('cp850', 'Spanish OEM (secondary)'), 
  55          '\x15' : ('cp437', 'Swedish OEM'), 
  56          '\x16' : ('cp850', 'Swedish OEM (secondary)'), 
  57          '\x17' : ('cp865', 'Norwegian OEM'), 
  58          '\x18' : ('cp437', 'Spanish OEM'), 
  59          '\x19' : ('cp437', 'English OEM (Britain)'), 
  60          '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'), 
  61          '\x1B' : ('cp437', 'English OEM (U.S.)'), 
  62          '\x1C' : ('cp863', 'French OEM (Canada)'), 
  63          '\x1D' : ('cp850', 'French OEM (secondary)'), 
  64          '\x1F' : ('cp852', 'Czech OEM'), 
  65          '\x22' : ('cp852', 'Hungarian OEM'), 
  66          '\x23' : ('cp852', 'Polish OEM'), 
  67          '\x24' : ('cp860', 'Portugese OEM'), 
  68          '\x25' : ('cp850', 'Potugese OEM (secondary)'), 
  69          '\x26' : ('cp866', 'Russian OEM'), 
  70          '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'), 
  71          '\x40' : ('cp852', 'Romanian OEM'), 
  72          '\x4D' : ('cp936', 'Chinese GBK (PRC)'), 
  73          '\x4E' : ('cp949', 'Korean (ANSI/OEM)'), 
  74          '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'), 
  75          '\x50' : ('cp874', 'Thai (ANSI/OEM)'), 
  76          '\x57' : ('cp1252', 'ANSI'), 
  77          '\x58' : ('cp1252', 'Western European ANSI'), 
  78          '\x59' : ('cp1252', 'Spanish ANSI'), 
  79          '\x64' : ('cp852', 'Eastern European MS-DOS'), 
  80          '\x65' : ('cp866', 'Russian MS-DOS'), 
  81          '\x66' : ('cp865', 'Nordic MS-DOS'), 
  82          '\x67' : ('cp861', 'Icelandic MS-DOS'), 
  83          '\x68' : (None, 'Kamenicky (Czech) MS-DOS'), 
  84          '\x69' : (None, 'Mazovia (Polish) MS-DOS'), 
  85          '\x6a' : ('cp737', 'Greek MS-DOS (437G)'), 
  86          '\x6b' : ('cp857', 'Turkish MS-DOS'), 
  87          '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'), 
  88          '\x79' : ('cp949', 'Korean Windows'), 
  89          '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'), 
  90          '\x7b' : ('cp932', 'Japanese Windows'), 
  91          '\x7c' : ('cp874', 'Thai Windows'), 
  92          '\x7d' : ('cp1255', 'Hebrew Windows'), 
  93          '\x7e' : ('cp1256', 'Arabic Windows'), 
  94          '\xc8' : ('cp1250', 'Eastern European Windows'), 
  95          '\xc9' : ('cp1251', 'Russian Windows'), 
  96          '\xca' : ('cp1254', 'Turkish Windows'), 
  97          '\xcb' : ('cp1253', 'Greek Windows'), 
  98          '\x96' : ('mac_cyrillic', 'Russian Macintosh'), 
  99          '\x97' : ('mac_latin2', 'Macintosh EE'), 
 100          '\x98' : ('mac_greek', 'Greek Macintosh') } 
 101   
 102  if sys.version_info[:2] < (2, 6): 
103 # define our own property type 104 - class property(object):
105 "Emulate PyProperty_Type() in Objects/descrobject.c" 106
107 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
108 self.fget = fget 109 self.fset = fset 110 self.fdel = fdel 111 self.__doc__ = doc or fget.__doc__
112 - def __call__(self, func):
113 self.fget = func 114 if not self.__doc__: 115 self.__doc__ = fget.__doc__
116 - def __get__(self, obj, objtype=None):
117 if obj is None: 118 return self 119 if self.fget is None: 120 raise AttributeError, "unreadable attribute" 121 return self.fget(obj)
122 - def __set__(self, obj, value):
123 if self.fset is None: 124 raise AttributeError, "can't set attribute" 125 self.fset(obj, value)
126 - def __delete__(self, obj):
127 if self.fdel is None: 128 raise AttributeError, "can't delete attribute" 129 self.fdel(obj)
130 - def setter(self, func):
131 self.fset = func 132 return self
133 - def deleter(self, func):
134 self.fdel = func 135 return self
136 # Internal classes
137 -class _DbfRecord(object):
138 """Provides routines to extract and save data within the fields of a dbf record.""" 139 __slots__ = ['_recnum', '_layout', '_data', '_dirty', '__weakref__']
140 - def _retrieveFieldValue(yo, record_data, fielddef):
141 """calls appropriate routine to fetch value stored in field from array 142 @param record_data: the data portion of the record 143 @type record_data: array of characters 144 @param fielddef: description of the field definition 145 @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags' 146 @returns: python data stored in field""" 147 148 field_type = fielddef['type'] 149 classtype = yo._layout.fieldtypes[field_type]['Class'] 150 retrieve = yo._layout.fieldtypes[field_type]['Retrieve'] 151 if classtype is not None: 152 datum = retrieve(record_data, fielddef, yo._layout.memo, classtype) 153 else: 154 datum = retrieve(record_data, fielddef, yo._layout.memo) 155 if field_type in yo._layout.character_fields: 156 datum = yo._layout.decoder(datum)[0] 157 if yo._layout.return_ascii: 158 try: 159 datum = yo._layout.output_encoder(datum)[0] 160 except UnicodeEncodeError: 161 datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore') 162 return datum
163 - def _updateFieldValue(yo, fielddef, value):
164 "calls appropriate routine to convert value to ascii bytes, and save it in record" 165 field_type = fielddef['type'] 166 update = yo._layout.fieldtypes[field_type]['Update'] 167 if field_type in yo._layout.character_fields: 168 if not isinstance(value, unicode): 169 if yo._layout.input_decoder is None: 170 raise NonUnicode("String not in unicode format, no default encoding specified") 171 value = yo._layout.input_decoder(value)[0] # input ascii => unicode 172 value = yo._layout.encoder(value)[0] # unicode => table ascii 173 bytes = array('c', update(value, fielddef, yo._layout.memo)) 174 size = fielddef['length'] 175 if len(bytes) > size: 176 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size)) 177 blank = array('c', ' ' * size) 178 start = fielddef['start'] 179 end = start + size 180 blank[:len(bytes)] = bytes[:] 181 yo._data[start:end] = blank[:] 182 yo._dirty = True
183 - def _update_disk(yo, location='', data=None):
184 if not yo._layout.inmemory: 185 if yo._recnum < 0: 186 raise DbfError("Attempted to update record that has been packed") 187 if location == '': 188 location = yo._recnum * yo._layout.header.record_length + yo._layout.header.start 189 if data is None: 190 data = yo._data 191 yo._layout.dfd.seek(location) 192 yo._layout.dfd.write(data) 193 yo._dirty = False 194 for index in yo.record_table._indexen: 195 index(yo)
196 - def __contains__(yo, key):
197 return key in yo._layout.fields or key in ['record_number','delete_flag']
198 - def __iter__(yo):
199 return (yo[field] for field in yo._layout.fields)
200 - def __getattr__(yo, name):
201 if name[0:2] == '__' and name[-2:] == '__': 202 raise AttributeError, 'Method %s is not implemented.' % name 203 elif name == 'record_number': 204 return yo._recnum 205 elif name == 'delete_flag': 206 return yo._data[0] != ' ' 207 elif not name in yo._layout.fields: 208 raise FieldMissing(name) 209 try: 210 fielddef = yo._layout[name] 211 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef) 212 return value 213 except DbfError, error: 214 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 215 raise
216 - def __getitem__(yo, item):
217 if type(item) in (int, long): 218 if not -yo._layout.header.field_count <= item < yo._layout.header.field_count: 219 raise IndexError("Field offset %d is not in record" % item) 220 return yo[yo._layout.fields[item]] 221 elif type(item) == slice: 222 sequence = [] 223 for index in yo._layout.fields[item]: 224 sequence.append(yo[index]) 225 return sequence 226 elif type(item) == str: 227 return yo.__getattr__(item) 228 else: 229 raise TypeError("%s is not a field name" % item)
230 - def __len__(yo):
231 return yo._layout.header.field_count
232 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
233 """record = ascii array of entire record; layout=record specification; memo = memo object for table""" 234 record = object.__new__(cls) 235 record._dirty = False 236 record._recnum = recnum 237 record._layout = layout 238 if layout.blankrecord is None and not _fromdisk: 239 record._createBlankRecord() 240 record._data = layout.blankrecord 241 if recnum == -1: # not a disk-backed record 242 return record 243 elif type(kamikaze) == array: 244 record._data = kamikaze[:] 245 elif type(kamikaze) == str: 246 record._data = array('c', kamikaze) 247 else: 248 record._data = kamikaze._data[:] 249 datalen = len(record._data) 250 if datalen < layout.header.record_length: 251 record._data.extend(layout.blankrecord[datalen:]) 252 elif datalen > layout.header.record_length: 253 record._data = record._data[:layout.header.record_length] 254 if not _fromdisk and not layout.inmemory: 255 record._update_disk() 256 return record
257 - def __setattr__(yo, name, value):
258 if name in yo.__slots__: 259 object.__setattr__(yo, name, value) 260 return 261 elif not name in yo._layout.fields: 262 raise FieldMissing(name) 263 fielddef = yo._layout[name] 264 try: 265 yo._updateFieldValue(fielddef, value) 266 except DbfError, error: 267 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 268 error.data = name 269 raise
270 - def __setitem__(yo, name, value):
271 if type(name) == str: 272 yo.__setattr__(name, value) 273 elif type(name) in (int, long): 274 yo.__setattr__(yo._layout.fields[name], value) 275 elif type(name) == slice: 276 sequence = [] 277 for field in yo._layout.fields[name]: 278 sequence.append(field) 279 if len(sequence) != len(value): 280 raise DbfError("length of slices not equal") 281 for field, val in zip(sequence, value): 282 yo[field] = val 283 else: 284 raise TypeError("%s is not a field name" % name)
285 - def __str__(yo):
286 result = [] 287 for seq, field in enumerate(yo.field_names): 288 result.append("%3d - %-10s: %s" % (seq, field, yo[field])) 289 return '\n'.join(result)
290 - def __repr__(yo):
291 return yo._data.tostring()
292 - def _createBlankRecord(yo):
293 "creates a blank record data chunk" 294 layout = yo._layout 295 ondisk = layout.ondisk 296 layout.ondisk = False 297 yo._data = array('c', ' ' * layout.header.record_length) 298 layout.memofields = [] 299 for field in layout.fields: 300 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']()) 301 if layout[field]['type'] in layout.memotypes: 302 layout.memofields.append(field) 303 layout.blankrecord = yo._data[:] 304 layout.ondisk = ondisk
305 - def delete_record(yo):
306 "marks record as deleted" 307 yo._data[0] = '*' 308 yo._dirty = True 309 return yo
310 @property
311 - def field_names(yo):
312 "fields in table/record" 313 return yo._layout.fields[:]
314 - def gather_fields(yo, dictionary, drop=False): # dict, drop_missing=False):
315 "saves a dictionary into a record's fields\nkeys with no matching field will raise a FieldMissing exception unless drop_missing = True" 316 old_data = yo._data[:] 317 try: 318 for key in dictionary: 319 if not key in yo.field_names: 320 if drop: 321 continue 322 raise FieldMissing(key) 323 yo.__setattr__(key, dictionary[key]) 324 except: 325 yo._data[:] = old_data 326 raise 327 return yo
328 @property
329 - def has_been_deleted(yo):
330 "marked for deletion?" 331 return yo._data[0] == '*'
332 - def read_record(yo):
333 "refresh record data from disk" 334 size = yo._layout.header.record_length 335 location = yo._recnum * size + yo._layout.header.start 336 yo._layout.dfd.seek(location) 337 yo._data[:] = yo._meta.dfd.read(size) 338 yo._dirty = False 339 return yo
340 @property
341 - def record_number(yo):
342 "physical record number" 343 return yo._recnum
344 @property
345 - def record_table(yo):
346 table = yo._layout.table() 347 if table is None: 348 raise DbfError("table is no longer available") 349 return table
350 - def check_index(yo):
351 for dbfindex in yo._layout.table()._indexen: 352 dbfindex(yo)
353 - def reset_record(yo, keep_fields=None):
354 "blanks record" 355 if keep_fields is None: 356 keep_fields = [] 357 keep = {} 358 for field in keep_fields: 359 keep[field] = yo[field] 360 if yo._layout.blankrecord == None: 361 yo._createBlankRecord() 362 yo._data[:] = yo._layout.blankrecord[:] 363 for field in keep_fields: 364 yo[field] = keep[field] 365 yo._dirty = True 366 return yo
367 - def scatter_fields(yo, blank=False):
368 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty." 369 keys = yo._layout.fields 370 if blank: 371 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys] 372 else: 373 values = [yo[field] for field in keys] 374 return dict(zip(keys, values))
375 - def undelete_record(yo):
376 "marks record as active" 377 yo._data[0] = ' ' 378 yo._dirty = True 379 return yo
380 - def write_record(yo, **kwargs):
381 "write record data to disk" 382 if kwargs: 383 yo.gather_fields(kwargs) 384 if yo._dirty: 385 yo._update_disk() 386 return 1 387 return 0
388 -class _DbfMemo(object):
389 """Provides access to memo fields as dictionaries 390 must override _init, _get_memo, and _put_memo to 391 store memo contents to disk"""
392 - def _init(yo):
393 "initialize disk file usage"
394 - def _get_memo(yo, block):
395 "retrieve memo contents from disk"
396 - def _put_memo(yo, data):
397 "store memo contents to disk"
398 - def __init__(yo, meta):
399 "" 400 yo.meta = meta 401 yo.memory = {} 402 yo.nextmemo = 1 403 yo._init() 404 yo.meta.newmemofile = False
405 - def get_memo(yo, block, field):
406 "gets the memo in block" 407 if yo.meta.ignorememos or not block: 408 return '' 409 if yo.meta.ondisk: 410 return yo._get_memo(block) 411 else: 412 return yo.memory[block]
413 - def put_memo(yo, data):
414 "stores data in memo file, returns block number" 415 if yo.meta.ignorememos or data == '': 416 return 0 417 if yo.meta.inmemory: 418 thismemo = yo.nextmemo 419 yo.nextmemo += 1 420 yo.memory[thismemo] = data 421 else: 422 thismemo = yo._put_memo(data) 423 return thismemo
424 -class _Db3Memo(_DbfMemo):
425 - def _init(yo):
426 "dBase III specific" 427 yo.meta.memo_size= 512 428 yo.record_header_length = 2 429 if yo.meta.ondisk and not yo.meta.ignorememos: 430 if yo.meta.newmemofile: 431 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 432 yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508) 433 else: 434 try: 435 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 436 yo.meta.mfd.seek(0) 437 yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4)) 438 except: 439 raise DbfError("memo file appears to be corrupt")
440 - def _get_memo(yo, block):
441 block = int(block) 442 yo.meta.mfd.seek(block * yo.meta.memo_size) 443 eom = -1 444 data = '' 445 while eom == -1: 446 newdata = yo.meta.mfd.read(yo.meta.memo_size) 447 if not newdata: 448 return data 449 data += newdata 450 eom = data.find('\x1a\x1a') 451 return data[:eom].rstrip()
452 - def _put_memo(yo, data):
453 data = data.rstrip() 454 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 455 blocks = length // yo.meta.memo_size 456 if length % yo.meta.memo_size: 457 blocks += 1 458 thismemo = yo.nextmemo 459 yo.nextmemo = thismemo + blocks 460 yo.meta.mfd.seek(0) 461 yo.meta.mfd.write(io.packLongInt(yo.nextmemo)) 462 yo.meta.mfd.seek(thismemo * yo.meta.memo_size) 463 yo.meta.mfd.write(data) 464 yo.meta.mfd.write('\x1a\x1a') 465 double_check = yo._get_memo(thismemo) 466 if len(double_check) != len(data): 467 uhoh = open('dbf_memo_dump.err','wb') 468 uhoh.write('thismemo: %d' % thismemo) 469 uhoh.write('nextmemo: %d' % yo.nextmemo) 470 uhoh.write('saved: %d bytes' % len(data)) 471 uhoh.write(data) 472 uhoh.write('retrieved: %d bytes' % len(double_check)) 473 uhoh.write(double_check) 474 uhoh.close() 475 raise DbfError("unknown error: memo not saved") 476 return thismemo
477 -class _VfpMemo(_DbfMemo):
478 - def _init(yo):
479 "Visual Foxpro 6 specific" 480 if yo.meta.ondisk and not yo.meta.ignorememos: 481 yo.record_header_length = 8 482 if yo.meta.newmemofile: 483 if yo.meta.memo_size == 0: 484 yo.meta.memo_size = 1 485 elif 1 < yo.meta.memo_size < 33: 486 yo.meta.memo_size *= 512 487 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 488 nextmemo = 512 // yo.meta.memo_size 489 if nextmemo * yo.meta.memo_size < 512: 490 nextmemo += 1 491 yo.nextmemo = nextmemo 492 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \ 493 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504) 494 else: 495 try: 496 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 497 yo.meta.mfd.seek(0) 498 header = yo.meta.mfd.read(512) 499 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True) 500 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True) 501 except: 502 raise DbfError("memo file appears to be corrupt")
503 - def _get_memo(yo, block):
504 yo.meta.mfd.seek(block * yo.meta.memo_size) 505 header = yo.meta.mfd.read(8) 506 length = io.unpackLongInt(header[4:], bigendian=True) 507 return yo.meta.mfd.read(length)
508 - def _put_memo(yo, data):
509 data = data.rstrip() # no trailing whitespace 510 yo.meta.mfd.seek(0) 511 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True) 512 yo.meta.mfd.seek(0) 513 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 514 blocks = length // yo.meta.memo_size 515 if length % yo.meta.memo_size: 516 blocks += 1 517 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True)) 518 yo.meta.mfd.seek(thismemo*yo.meta.memo_size) 519 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data) 520 return thismemo
521 # Public classes
522 -class DbfTable(object):
523 """Provides a framework for dbf style tables.""" 524 _version = 'basic memory table' 525 _versionabbv = 'dbf' 526 _fieldtypes = { 527 'D' : { 'Type':'Date', 'Init':io.addDate, 'Blank':Date.today, 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Class':None}, 528 'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool, 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Class':None}, 529 'M' : { 'Type':'Memo', 'Init':io.addMemo, 'Blank':str, 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Class':None} } 530 _memoext = '' 531 _memotypes = tuple('M', ) 532 _memoClass = _DbfMemo 533 _yesMemoMask = '' 534 _noMemoMask = '' 535 _fixed_fields = ('M','D','L') # always same length in table 536 _variable_fields = tuple() # variable length in table 537 _character_fields = tuple('M', ) # field representing character data 538 _decimal_fields = tuple() # text-based numeric fields 539 _numeric_fields = tuple() # fields representing a number 540 _currency_fields = tuple() 541 _dbfTableHeader = array('c', '\x00' * 32) 542 _dbfTableHeader[0] = '\x00' # table type - none 543 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 544 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 545 _dbfTableHeader[29] = '\x00' # code page -- none, using plain ascii 546 _dbfTableHeader = _dbfTableHeader.tostring() 547 _dbfTableHeaderExtra = '' 548 _supported_tables = [] 549 _read_only = False 550 _meta_only = False 551 _use_deleted = True 552 backup = False
553 - class _DbfLists(object):
554 "implements the weakref structure for DbfLists"
555 - def __init__(yo):
556 yo._lists = set()
557 - def __iter__(yo):
558 yo._lists = set([s for s in yo._lists if s() is not None]) 559 return (s() for s in yo._lists if s() is not None)
560 - def __len__(yo):
561 yo._lists = set([s for s in yo._lists if s() is not None]) 562 return len(yo._lists)
563 - def add(yo, new_list):
564 yo._lists.add(weakref.ref(new_list)) 565 yo._lists = set([s for s in yo._lists if s() is not None])
566 - class _Indexen(object):
567 "implements the weakref structure for seperate indexes"
568 - def __init__(yo):
569 yo._indexen = set()
570 - def __iter__(yo):
571 yo._indexen = set([s for s in yo._indexen if s() is not None]) 572 return (s() for s in yo._indexen if s() is not None)
573 - def __len__(yo):
574 yo._indexen = set([s for s in yo._indexen if s() is not None]) 575 return len(yo._indexen)
576 - def add(yo, new_list):
577 yo._indexen.add(weakref.ref(new_list)) 578 yo._indexen = set([s for s in yo._indexen if s() is not None])
579 - class _MetaData(dict):
580 blankrecord = None 581 fields = None 582 filename = None 583 dfd = None 584 memoname = None 585 newmemofile = False 586 memo = None 587 mfd = None 588 ignorememos = False 589 memofields = None 590 current = -1
591 - class _TableHeader(object):
592 - def __init__(yo, data):
593 if len(data) != 32: 594 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data)) 595 yo._data = array('c', data + '\x0d')
596 - def codepage(yo, cp=None):
597 "get/set code page of table" 598 if cp is None: 599 return yo._data[29] 600 else: 601 cp, sd, ld = _codepage_lookup(cp) 602 yo._data[29] = cp 603 return cp
604 @property
605 - def data(yo):
606 "main data structure" 607 date = io.packDate(Date.today()) 608 yo._data[1:4] = array('c', date) 609 return yo._data.tostring()
610 @data.setter
611 - def data(yo, bytes):
612 if len(bytes) < 32: 613 raise DbfError("length for data of %d is less than 32" % len(bytes)) 614 yo._data[:] = array('c', bytes)
615 @property
616 - def extra(yo):
617 "extra dbf info (located after headers, before data records)" 618 fieldblock = yo._data[32:] 619 for i in range(len(fieldblock)//32+1): 620 cr = i * 32 621 if fieldblock[cr] == '\x0d': 622 break 623 else: 624 raise DbfError("corrupt field structure") 625 cr += 33 # skip past CR 626 return yo._data[cr:].tostring()
627 @extra.setter
628 - def extra(yo, data):
629 fieldblock = yo._data[32:] 630 for i in range(len(fieldblock)//32+1): 631 cr = i * 32 632 if fieldblock[cr] == '\x0d': 633 break 634 else: 635 raise DbfError("corrupt field structure") 636 cr += 33 # skip past CR 637 yo._data[cr:] = array('c', data) # extra 638 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start
639 @property
640 - def field_count(yo):
641 "number of fields (read-only)" 642 fieldblock = yo._data[32:] 643 for i in range(len(fieldblock)//32+1): 644 cr = i * 32 645 if fieldblock[cr] == '\x0d': 646 break 647 else: 648 raise DbfError("corrupt field structure") 649 return len(fieldblock[:cr]) // 32
650 @property
651 - def fields(yo):
652 "field block structure" 653 fieldblock = yo._data[32:] 654 for i in range(len(fieldblock)//32+1): 655 cr = i * 32 656 if fieldblock[cr] == '\x0d': 657 break 658 else: 659 raise DbfError("corrupt field structure") 660 return fieldblock[:cr].tostring()
661 @fields.setter
662 - def fields(yo, block):
663 fieldblock = yo._data[32:] 664 for i in range(len(fieldblock)//32+1): 665 cr = i * 32 666 if fieldblock[cr] == '\x0d': 667 break 668 else: 669 raise DbfError("corrupt field structure") 670 cr += 32 # convert to indexing main structure 671 fieldlen = len(block) 672 if fieldlen % 32 != 0: 673 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen) 674 yo._data[32:cr] = array('c', block) # fields 675 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start 676 fieldlen = fieldlen // 32 677 recordlen = 1 # deleted flag 678 for i in range(fieldlen): 679 recordlen += ord(block[i*32+16]) 680 yo._data[10:12] = array('c', io.packShortInt(recordlen))
681 @property
682 - def record_count(yo):
683 "number of records (maximum 16,777,215)" 684 return io.unpackLongInt(yo._data[4:8].tostring())
685 @record_count.setter
686 - def record_count(yo, count):
687 yo._data[4:8] = array('c', io.packLongInt(count))
688 @property
689 - def record_length(yo):
690 "length of a record (read_only) (max of 65,535)" 691 return io.unpackShortInt(yo._data[10:12].tostring())
692 @property
693 - def start(yo):
694 "starting position of first record in file (must be within first 64K)" 695 return io.unpackShortInt(yo._data[8:10].tostring())
696 @start.setter
697 - def start(yo, pos):
698 yo._data[8:10] = array('c', io.packShortInt(pos))
699 @property
700 - def update(yo):
701 "date of last table modification (read-only)" 702 return io.unpackDate(yo._data[1:4].tostring())
703 @property
704 - def version(yo):
705 "dbf version" 706 return yo._data[0]
707 @version.setter
708 - def version(yo, ver):
709 yo._data[0] = ver
710 - class _Table(object):
711 "implements the weakref table for records"
712 - def __init__(yo, count, meta):
713 yo._meta = meta 714 yo._weakref_list = [weakref.ref(lambda x: None)] * count
715 - def __getitem__(yo, index):
716 maybe = yo._weakref_list[index]() 717 if maybe is None: 718 if index < 0: 719 index += yo._meta.header.record_count 720 size = yo._meta.header.record_length 721 location = index * size + yo._meta.header.start 722 yo._meta.dfd.seek(location) 723 if yo._meta.dfd.tell() != location: 724 raise ValueError("unable to seek to offset %d in file" % location) 725 bytes = yo._meta.dfd.read(size) 726 if not bytes: 727 raise ValueError("unable to read record data from %s at location %d" % (yo._meta.filename, location)) 728 maybe = _DbfRecord(recnum=index, layout=yo._meta, kamikaze=bytes, _fromdisk=True) 729 yo._weakref_list[index] = weakref.ref(maybe) 730 return maybe
731 - def append(yo, record):
732 yo._weakref_list.append(weakref.ref(record))
733 - def clear(yo):
734 yo._weakref_list[:] = []
735 - def pop(yo):
736 return yo._weakref_list.pop()
737 - class DbfIterator(object):
738 "returns records using current index"
739 - def __init__(yo, table):
740 yo._table = table 741 yo._index = -1 742 yo._more_records = True
743 - def __iter__(yo):
744 return yo
745 - def next(yo):
746 while yo._more_records: 747 yo._index += 1 748 if yo._index >= len(yo._table): 749 yo._more_records = False 750 continue 751 record = yo._table[yo._index] 752 if not yo._table.use_deleted and record.has_been_deleted: 753 continue 754 return record 755 else: 756 raise StopIteration
757 - def _buildHeaderFields(yo):
758 "constructs fieldblock for disk table" 759 fieldblock = array('c', '') 760 memo = False 761 yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask)) 762 for field in yo._meta.fields: 763 if yo._meta.fields.count(field) > 1: 764 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)") 765 fielddef = array('c', '\x00' * 32) 766 fielddef[:11] = array('c', io.packStr(field)) 767 fielddef[11] = yo._meta[field]['type'] 768 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start'])) 769 fielddef[16] = chr(yo._meta[field]['length']) 770 fielddef[17] = chr(yo._meta[field]['decimals']) 771 fielddef[18] = chr(yo._meta[field]['flags']) 772 fieldblock.extend(fielddef) 773 if yo._meta[field]['type'] in yo._meta.memotypes: 774 memo = True 775 yo._meta.header.fields = fieldblock.tostring() 776 if memo: 777 yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask)) 778 if yo._meta.memo is None: 779 yo._meta.memo = yo._memoClass(yo._meta)
780 - def _checkMemoIntegrity(yo):
781 "dBase III specific" 782 if yo._meta.header.version == '\x83': 783 try: 784 yo._meta.memo = yo._memoClass(yo._meta) 785 except: 786 yo._meta.dfd.close() 787 yo._meta.dfd = None 788 raise 789 if not yo._meta.ignorememos: 790 for field in yo._meta.fields: 791 if yo._meta[field]['type'] in yo._memotypes: 792 if yo._meta.header.version != '\x83': 793 yo._meta.dfd.close() 794 yo._meta.dfd = None 795 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 796 elif not os.path.exists(yo._meta.memoname): 797 yo._meta.dfd.close() 798 yo._meta.dfd = None 799 raise DbfError("Table structure corrupt: memo fields exist without memo file") 800 break
801 - def _initializeFields(yo):
802 "builds the FieldList of names, types, and descriptions from the disk file" 803 yo._meta.fields[:] = [] 804 offset = 1 805 fieldsdef = yo._meta.header.fields 806 if len(fieldsdef) % 32 != 0: 807 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 808 if len(fieldsdef) // 32 != yo.field_count: 809 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 810 for i in range(yo.field_count): 811 fieldblock = fieldsdef[i*32:(i+1)*32] 812 name = io.unpackStr(fieldblock[:11]) 813 type = fieldblock[11] 814 if not type in yo._meta.fieldtypes: 815 raise DbfError("Unknown field type: %s" % type) 816 start = offset 817 length = ord(fieldblock[16]) 818 offset += length 819 end = start + length 820 decimals = ord(fieldblock[17]) 821 flags = ord(fieldblock[18]) 822 if name in yo._meta.fields: 823 raise DbfError('Duplicate field name found: %s' % name) 824 yo._meta.fields.append(name) 825 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
826 - def _fieldLayout(yo, i):
827 "Returns field information Name Type(Length[,Decimals])" 828 name = yo._meta.fields[i] 829 type = yo._meta[name]['type'] 830 length = yo._meta[name]['length'] 831 decimals = yo._meta[name]['decimals'] 832 if type in yo._decimal_fields: 833 description = "%s %s(%d,%d)" % (name, type, length, decimals) 834 elif type in yo._fixed_fields: 835 description = "%s %s" % (name, type) 836 else: 837 description = "%s %s(%d)" % (name, type, length) 838 return description
839 - def _loadtable(yo):
840 "loads the records from disk to memory" 841 if yo._meta_only: 842 raise DbfError("%s has been closed, records are unavailable" % yo.filename) 843 dfd = yo._meta.dfd 844 header = yo._meta.header 845 dfd.seek(header.start) 846 allrecords = dfd.read() # kludge to get around mysterious errno 0 problems 847 dfd.seek(0) 848 length = header.record_length 849 for i in range(header.record_count): 850 record_data = allrecords[length*i:length*i+length] 851 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True)) 852 dfd.seek(0)
853 - def _list_fields(yo, specs, sep=','):
854 if specs is None: 855 specs = yo.field_names 856 elif isinstance(specs, str): 857 specs = specs.split(sep) 858 else: 859 specs = list(specs) 860 specs = [s.strip() for s in specs] 861 return specs
862 - def _update_disk(yo, headeronly=False):
863 "synchronizes the disk file with current data" 864 if yo._meta.inmemory: 865 return 866 fd = yo._meta.dfd 867 fd.seek(0) 868 fd.write(yo._meta.header.data) 869 if not headeronly: 870 for record in yo._table: 871 record._update_disk() 872 fd.flush() 873 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length) 874 if 'db3' in yo._versionabbv: 875 fd.seek(0, os.SEEK_END) 876 fd.write('\x1a') # required for dBase III 877 fd.flush() 878 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length + 1)
879
880 - def __contains__(yo, key):
881 return key in yo.field_names
882 - def __enter__(yo):
883 return yo
884 - def __exit__(yo, *exc_info):
885 yo.close()
886 - def __getattr__(yo, name):
887 if name in ('_table'): 888 if yo._meta.ondisk: 889 yo._table = yo._Table(len(yo), yo._meta) 890 else: 891 yo._table = [] 892 yo._loadtable() 893 return object.__getattribute__(yo, name)
894 - def __getitem__(yo, value):
895 if type(value) == int: 896 if not -yo._meta.header.record_count <= value < yo._meta.header.record_count: 897 raise IndexError("Record %d is not in table." % value) 898 return yo._table[value] 899 elif type(value) == slice: 900 sequence = List(desc='%s --> %s' % (yo.filename, value), field_names=yo.field_names) 901 yo._dbflists.add(sequence) 902 for index in range(len(yo))[value]: 903 record = yo._table[index] 904 if yo.use_deleted is True or not record.has_been_deleted: 905 sequence.append(record) 906 return sequence 907 else: 908 raise TypeError('type <%s> not valid for indexing' % type(value))
909 - def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False, 910 read_only=False, keep_memos=False, meta_only=False, codepage=None, 911 numbers='default', strings=str, currency=Decimal):
912 """open/create dbf file 913 filename should include path if needed 914 field_specs can be either a ;-delimited string or a list of strings 915 memo_size is always 512 for db3 memos 916 ignore_memos is useful if the memo file is missing or corrupt 917 read_only will load records into memory, then close the disk file 918 keep_memos will also load any memo fields into memory 919 meta_only will ignore all records, keeping only basic table information 920 codepage will override whatever is set in the table itself""" 921 if filename[0] == filename[-1] == ':': 922 if field_specs is None: 923 raise DbfError("field list must be specified for memory tables") 924 elif type(yo) is DbfTable: 925 raise DbfError("only memory tables supported") 926 yo._dbflists = yo._DbfLists() 927 yo._indexen = yo._Indexen() 928 yo._meta = meta = yo._MetaData() 929 for datatypes, classtype in ( 930 (yo._character_fields, strings), 931 (yo._numeric_fields, numbers), 932 (yo._currency_fields, currency), 933 ): 934 for datatype in datatypes: 935 yo._fieldtypes[datatype]['Class'] = classtype 936 meta.numbers = numbers 937 meta.strings = strings 938 meta.currency = currency 939 meta.table = weakref.ref(yo) 940 meta.filename = filename 941 meta.fields = [] 942 meta.fieldtypes = yo._fieldtypes 943 meta.fixed_fields = yo._fixed_fields 944 meta.variable_fields = yo._variable_fields 945 meta.character_fields = yo._character_fields 946 meta.decimal_fields = yo._decimal_fields 947 meta.numeric_fields = yo._numeric_fields 948 meta.memotypes = yo._memotypes 949 meta.ignorememos = ignore_memos 950 meta.memo_size = memo_size 951 meta.input_decoder = codecs.getdecoder(input_decoding) # from ascii to unicode 952 meta.output_encoder = codecs.getencoder(input_decoding) # and back to ascii 953 meta.return_ascii = return_ascii 954 meta.header = header = yo._TableHeader(yo._dbfTableHeader) 955 header.extra = yo._dbfTableHeaderExtra 956 header.data #force update of date 957 if filename[0] == filename[-1] == ':': 958 yo._table = [] 959 meta.ondisk = False 960 meta.inmemory = True 961 meta.memoname = filename 962 else: 963 base, ext = os.path.splitext(filename) 964 if ext == '': 965 meta.filename = base + '.dbf' 966 meta.memoname = base + yo._memoext 967 meta.ondisk = True 968 meta.inmemory = False 969 if field_specs: 970 if meta.ondisk: 971 meta.dfd = open(meta.filename, 'w+b') 972 meta.newmemofile = True 973 yo.add_fields(field_specs) 974 header.codepage(codepage or default_codepage) 975 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 976 meta.decoder = codecs.getdecoder(sd) 977 meta.encoder = codecs.getencoder(sd) 978 return 979 try: 980 dfd = meta.dfd = open(meta.filename, 'r+b') 981 except IOError, e: 982 raise DbfError(str(e)) 983 dfd.seek(0) 984 meta.header = header = yo._TableHeader(dfd.read(32)) 985 if not header.version in yo._supported_tables: 986 dfd.close() 987 dfd = None 988 raise DbfError( 989 "%s does not support %s [%x]" % 990 (yo._version, 991 version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), 992 ord(meta.header.version))) 993 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 994 yo._meta.decoder = codecs.getdecoder(sd) 995 yo._meta.encoder = codecs.getencoder(sd) 996 fieldblock = dfd.read(header.start - 32) 997 for i in range(len(fieldblock)//32+1): 998 fieldend = i * 32 999 if fieldblock[fieldend] == '\x0d': 1000 break 1001 else: 1002 raise DbfError("corrupt field structure in header") 1003 if len(fieldblock[:fieldend]) % 32 != 0: 1004 raise DbfError("corrupt field structure in header") 1005 header.fields = fieldblock[:fieldend] 1006 header.extra = fieldblock[fieldend+1:] # skip trailing \r 1007 yo._initializeFields() 1008 yo._checkMemoIntegrity() 1009 meta.current = -1 1010 if len(yo) > 0: 1011 meta.current = 0 1012 dfd.seek(0) 1013 if meta_only: 1014 yo.close(keep_table=False, keep_memos=False) 1015 elif read_only: 1016 yo.close(keep_table=True, keep_memos=keep_memos) 1017 if codepage is not None: 1018 cp, sd, ld = _codepage_lookup(codepage) 1019 yo._meta.decoder = codecs.getdecoder(sd) 1020 yo._meta.encoder = codecs.getencoder(sd)
1021
1022 - def __iter__(yo):
1023 return yo.DbfIterator(yo)
1024 - def __len__(yo):
1025 return yo._meta.header.record_count
1026 - def __nonzero__(yo):
1027 return yo._meta.header.record_count != 0
1028 - def __repr__(yo):
1029 if yo._read_only: 1030 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename 1031 elif yo._meta_only: 1032 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename 1033 else: 1034 return __name__ + ".Table('%s')" % yo._meta.filename
1035 - def __str__(yo):
1036 if yo._read_only: 1037 status = "read-only" 1038 elif yo._meta_only: 1039 status = "meta-only" 1040 else: 1041 status = "read/write" 1042 str = """ 1043 Table: %s 1044 Type: %s 1045 Codepage: %s 1046 Status: %s 1047 Last updated: %s 1048 Record count: %d 1049 Field count: %d 1050 Record length: %d """ % (yo.filename, version_map.get(yo._meta.header.version, 1051 'unknown - ' + hex(ord(yo._meta.header.version))), yo.codepage, status, 1052 yo.last_update, len(yo), yo.field_count, yo.record_length) 1053 str += "\n --Fields--\n" 1054 for i in range(len(yo._meta.fields)): 1055 str += "%11d) %s\n" % (i, yo._fieldLayout(i)) 1056 return str
1057 @property
1058 - def codepage(yo):
1059 return "%s (%s)" % code_pages[yo._meta.header.codepage()]
1060 @codepage.setter
1061 - def codepage(yo, cp):
1062 cp = code_pages[yo._meta.header.codepage(cp)][0] 1063 yo._meta.decoder = codecs.getdecoder(cp) 1064 yo._meta.encoder = codecs.getencoder(cp) 1065 yo._update_disk(headeronly=True)
1066 @property
1067 - def field_count(yo):
1068 "the number of fields in the table" 1069 return yo._meta.header.field_count
1070 @property
1071 - def field_names(yo):
1072 "a list of the fields in the table" 1073 return yo._meta.fields[:]
1074 @property
1075 - def filename(yo):
1076 "table's file name, including path (if specified on open)" 1077 return yo._meta.filename
1078 @property
1079 - def last_update(yo):
1080 "date of last update" 1081 return yo._meta.header.update
1082 @property
1083 - def memoname(yo):
1084 "table's memo name (if path included in filename on open)" 1085 return yo._meta.memoname
1086 @property
1087 - def record_length(yo):
1088 "number of bytes in a record" 1089 return yo._meta.header.record_length
1090 @property
1091 - def record_number(yo):
1092 "index number of the current record" 1093 return yo._meta.current
1094 @property
1095 - def supported_tables(yo):
1096 "allowable table types" 1097 return yo._supported_tables
1098 @property
1099 - def use_deleted(yo):
1100 "process or ignore deleted records" 1101 return yo._use_deleted
1102 @use_deleted.setter
1103 - def use_deleted(yo, new_setting):
1104 yo._use_deleted = new_setting
1105 @property
1106 - def version(yo):
1107 "returns the dbf type of the table" 1108 return yo._version
1109 - def add_fields(yo, field_specs):
1110 """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]] 1111 backup table is created with _backup appended to name 1112 then modifies current structure""" 1113 all_records = [record for record in yo] 1114 if yo: 1115 yo.create_backup() 1116 yo._meta.blankrecord = None 1117 meta = yo._meta 1118 offset = meta.header.record_length 1119 fields = yo._list_fields(field_specs, sep=';') 1120 for field in fields: 1121 try: 1122 name, format = field.split() 1123 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum(): 1124 raise DbfError("%s invalid: field names must start with a letter, and can only contain letters, digits, and _" % name) 1125 name = name.lower() 1126 if name in meta.fields: 1127 raise DbfError("Field '%s' already exists" % name) 1128 field_type = format[0].upper() 1129 if len(name) > 10: 1130 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name))) 1131 if not field_type in meta.fieldtypes.keys(): 1132 raise DbfError("Unknown field type: %s" % field_type) 1133 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format) 1134 except ValueError: 1135 raise DbfError("invalid field specifier: %s (multiple fields should be separated with ';'" % field) 1136 start = offset 1137 end = offset + length 1138 offset = end 1139 meta.fields.append(name) 1140 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0} 1141 if meta[name]['type'] in yo._memotypes and meta.memo is None: 1142 meta.memo = yo._memoClass(meta) 1143 for record in yo: 1144 record[name] = meta.fieldtypes[field_type]['Blank']() 1145 yo._buildHeaderFields() 1146 yo._update_disk()
1147 - def append(yo, kamikaze='', drop=False, multiple=1):
1148 "adds <multiple> blank records, and fills fields with dict/tuple values if present" 1149 if not yo.field_count: 1150 raise DbfError("No fields defined, cannot append") 1151 empty_table = len(yo) == 0 1152 dictdata = False 1153 tupledata = False 1154 if not isinstance(kamikaze, _DbfRecord): 1155 if isinstance(kamikaze, dict): 1156 dictdata = kamikaze 1157 kamikaze = '' 1158 elif isinstance(kamikaze, tuple): 1159 tupledata = kamikaze 1160 kamikaze = '' 1161 newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze) 1162 yo._table.append(newrecord) 1163 yo._meta.header.record_count += 1 1164 try: 1165 if dictdata: 1166 newrecord.gather_fields(dictdata, drop=drop) 1167 elif tupledata: 1168 for index, item in enumerate(tupledata): 1169 newrecord[index] = item 1170 elif kamikaze == str: 1171 for field in yo._meta.memofields: 1172 newrecord[field] = '' 1173 elif kamikaze: 1174 for field in yo._meta.memofields: 1175 newrecord[field] = kamikaze[field] 1176 newrecord.write_record() 1177 except Exception: 1178 yo._table.pop() # discard failed record 1179 yo._meta.header.record_count = yo._meta.header.record_count - 1 1180 yo._update_disk() 1181 raise 1182 multiple -= 1 1183 if multiple: 1184 data = newrecord._data 1185 single = yo._meta.header.record_count 1186 total = single + multiple 1187 while single < total: 1188 multi_record = _DbfRecord(single, yo._meta, kamikaze=data) 1189 yo._table.append(multi_record) 1190 for field in yo._meta.memofields: 1191 multi_record[field] = newrecord[field] 1192 single += 1 1193 multi_record.write_record() 1194 yo._meta.header.record_count = total # += multiple 1195 yo._meta.current = yo._meta.header.record_count - 1 1196 newrecord = multi_record 1197 yo._update_disk(headeronly=True) 1198 if empty_table: 1199 yo._meta.current = 0 1200 return newrecord
1201 - def bof(yo, _move=False):
1202 "moves record pointer to previous usable record; returns True if no more usable records" 1203 current = yo._meta.current 1204 try: 1205 while yo._meta.current > 0: 1206 yo._meta.current -= 1 1207 if yo.use_deleted or not yo.current().has_been_deleted: 1208 break 1209 else: 1210 yo._meta.current = -1 1211 return True 1212 return False 1213 finally: 1214 if not _move: 1215 yo._meta.current = current
1216 - def bottom(yo, get_record=False):
1217 """sets record pointer to bottom of table 1218 if get_record, seeks to and returns last (non-deleted) record 1219 DbfError if table is empty 1220 Bof if all records deleted and use_deleted is False""" 1221 yo._meta.current = yo._meta.header.record_count 1222 if get_record: 1223 try: 1224 return yo.prev() 1225 except Bof: 1226 yo._meta.current = yo._meta.header.record_count 1227 raise Eof()
1228 - def close(yo, keep_table=False, keep_memos=False):
1229 """closes disk files 1230 ensures table data is available if keep_table 1231 ensures memo data is available if keep_memos""" 1232 yo._meta.inmemory = True 1233 if keep_table: 1234 replacement_table = [] 1235 for record in yo._table: 1236 replacement_table.append(record) 1237 yo._table = replacement_table 1238 else: 1239 if yo._meta.ondisk: 1240 yo._meta_only = True 1241 if yo._meta.mfd is not None: 1242 if not keep_memos: 1243 yo._meta.ignorememos = True 1244 else: 1245 memo_fields = [] 1246 for field in yo.field_names: 1247 if yo.is_memotype(field): 1248 memo_fields.append(field) 1249 for record in yo: 1250 for field in memo_fields: 1251 record[field] = record[field] 1252 yo._meta.mfd.close() 1253 yo._meta.mfd = None 1254 if yo._meta.ondisk: 1255 yo._meta.dfd.close() 1256 yo._meta.dfd = None 1257 if keep_table: 1258 yo._read_only = True 1259 yo._meta.ondisk = False
1260 - def create_backup(yo, new_name=None, overwrite=False):
1261 "creates a backup table -- ignored if memory table" 1262 if yo.filename[0] == yo.filename[-1] == ':': 1263 return 1264 if new_name is None: 1265 upper = yo.filename.isupper() 1266 name, ext = os.path.splitext(os.path.split(yo.filename)[1]) 1267 extra = '_BACKUP' if upper else '_backup' 1268 new_name = os.path.join(temp_dir, name + extra + ext) 1269 else: 1270 overwrite = True 1271 if overwrite or not yo.backup: 1272 bkup = open(new_name, 'wb') 1273 try: 1274 yo._meta.dfd.seek(0) 1275 copyfileobj(yo._meta.dfd, bkup) 1276 yo.backup = new_name 1277 finally: 1278 bkup.close()
1279 - def create_index(yo, key):
1280 return Index(yo, key)
1281 - def current(yo, index=False):
1282 "returns current logical record, or its index" 1283 if yo._meta.current < 0: 1284 raise Bof() 1285 elif yo._meta.current >= yo._meta.header.record_count: 1286 raise Eof() 1287 if index: 1288 return yo._meta.current 1289 return yo._table[yo._meta.current]
1290 - def delete_fields(yo, doomed):
1291 """removes field(s) from the table 1292 creates backup files with _backup appended to the file name, 1293 then modifies current structure""" 1294 doomed = yo._list_fields(doomed) 1295 for victim in doomed: 1296 if victim not in yo._meta.fields: 1297 raise DbfError("field %s not in table -- delete aborted" % victim) 1298 all_records = [record for record in yo] 1299 yo.create_backup() 1300 for victim in doomed: 1301 yo._meta.fields.pop(yo._meta.fields.index(victim)) 1302 start = yo._meta[victim]['start'] 1303 end = yo._meta[victim]['end'] 1304 for record in yo: 1305 record._data = record._data[:start] + record._data[end:] 1306 for field in yo._meta.fields: 1307 if yo._meta[field]['start'] == end: 1308 end = yo._meta[field]['end'] 1309 yo._meta[field]['start'] = start 1310 yo._meta[field]['end'] = start + yo._meta[field]['length'] 1311 start = yo._meta[field]['end'] 1312 yo._buildHeaderFields() 1313 yo._update_disk()
1314 - def eof(yo, _move=False):
1315 "moves record pointer to next usable record; returns True if no more usable records" 1316 current = yo._meta.current 1317 try: 1318 while yo._meta.current < yo._meta.header.record_count - 1: 1319 yo._meta.current += 1 1320 if yo.use_deleted or not yo.current().has_been_deleted: 1321 break 1322 else: 1323 yo._meta.current = yo._meta.header.record_count 1324 return True 1325 return False 1326 finally: 1327 if not _move: 1328 yo._meta.current = current
1329 - def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
1330 """writes the table using CSV or tab-delimited format, using the filename 1331 given if specified, otherwise the table name""" 1332 if filename is not None: 1333 path, filename = os.path.split(filename) 1334 else: 1335 path, filename = os.path.split(yo.filename) 1336 filename = os.path.join(path, filename) 1337 field_specs = yo._list_fields(field_specs) 1338 if records is None: 1339 records = yo 1340 format = format.lower() 1341 if format not in ('csv', 'tab', 'fixed'): 1342 raise DbfError("export format: csv, tab, or fixed -- not %s" % format) 1343 if format == 'fixed': 1344 format = 'txt' 1345 base, ext = os.path.splitext(filename) 1346 if ext.lower() in ('', '.dbf'): 1347 filename = base + "." + format[:3] 1348 fd = open(filename, 'w') 1349 try: 1350 if format == 'csv': 1351 csvfile = csv.writer(fd, dialect='dbf') 1352 if header: 1353 csvfile.writerow(field_specs) 1354 for record in records: 1355 fields = [] 1356 for fieldname in field_specs: 1357 fields.append(record[fieldname]) 1358 csvfile.writerow(fields) 1359 elif format == 'tab': 1360 if header: 1361 fd.write('\t'.join(field_specs) + '\n') 1362 for record in records: 1363 fields = [] 1364 for fieldname in field_specs: 1365 fields.append(str(record[fieldname])) 1366 fd.write('\t'.join(fields) + '\n') 1367 else: # format == 'fixed' 1368 header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w') 1369 header.write("%-15s Size\n" % "Field Name") 1370 header.write("%-15s ----\n" % ("-" * 15)) 1371 sizes = [] 1372 for field in field_specs: 1373 size = yo.size(field)[0] 1374 sizes.append(size) 1375 header.write("%-15s %3d\n" % (field, size)) 1376 header.write('\nTotal Records in file: %d\n' % len(records)) 1377 header.close() 1378 for record in records: 1379 fields = [] 1380 for i, field_name in enumerate(field_specs): 1381 fields.append("%-*s" % (sizes[i], record[field_name])) 1382 fd.write(''.join(fields) + '\n') 1383 finally: 1384 fd.close() 1385 fd = None 1386 return len(records)
1387 - def find(yo, command):
1388 "uses exec to perform queries on the table" 1389 possible = List(desc="%s --> %s" % (yo.filename, command), field_names=yo.field_names) 1390 yo._dbflists.add(possible) 1391 result = {} 1392 select = 'result["keep"] = %s' % command 1393 g = {} 1394 use_deleted = yo.use_deleted 1395 for record in yo: 1396 result['keep'] = False 1397 g['result'] = result 1398 exec select in g, record 1399 if result['keep']: 1400 possible.append(record) 1401 record.write_record() 1402 return possible
1403 - def get_record(yo, recno):
1404 "returns record at physical_index[recno]" 1405 return yo._table[recno]
1406 - def goto(yo, criteria):
1407 """changes the record pointer to the first matching (non-deleted) record 1408 criteria should be either a tuple of tuple(value, field, func) triples, 1409 or an integer to go to""" 1410 if isinstance(criteria, int): 1411 if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count: 1412 raise IndexError("Record %d does not exist" % criteria) 1413 if criteria < 0: 1414 criteria += yo._meta.header.record_count 1415 yo._meta.current = criteria 1416 return yo.current() 1417 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop]) 1418 specs = tuple([(field, func) for value, field, func in criteria]) 1419 match = tuple([value for value, field, func in criteria]) 1420 current = yo.current(index=True) 1421 matchlen = len(match) 1422 while not yo.Eof(): 1423 record = yo.current() 1424 results = record(*specs) 1425 if results == match: 1426 return record 1427 return yo.goto(current)
1428 - def is_decimal(yo, name):
1429 "returns True if name is a variable-length field type" 1430 return yo._meta[name]['type'] in yo._decimal_fields
1431 - def is_memotype(yo, name):
1432 "returns True if name is a memo type field" 1433 return yo._meta[name]['type'] in yo._memotypes
1434 - def new(yo, filename, field_specs=None, codepage=None):
1435 "returns a new table of the same type" 1436 if field_specs is None: 1437 field_specs = yo.structure() 1438 if not (filename[0] == filename[-1] == ':'): 1439 path, name = os.path.split(filename) 1440 if path == "": 1441 filename = os.path.join(os.path.split(yo.filename)[0], filename) 1442 elif name == "": 1443 filename = os.path.join(path, os.path.split(yo.filename)[1]) 1444 if codepage is None: 1445 codepage = yo._meta.header.codepage()[0] 1446 return yo.__class__(filename, field_specs, codepage=codepage)
1447 - def next(yo):
1448 "set record pointer to next (non-deleted) record, and return it" 1449 if yo.eof(_move=True): 1450 raise Eof() 1451 return yo.current()
1452 - def open(yo):
1453 meta = yo._meta 1454 meta.inmemory = False 1455 meta.ondisk = True 1456 yo._read_only = False 1457 yo._meta_only = False 1458 if '_table' in dir(yo): 1459 del yo._table 1460 dfd = meta.dfd = open(meta.filename, 'r+b') 1461 dfd.seek(0) 1462 meta.header = header = yo._TableHeader(dfd.read(32)) 1463 if not header.version in yo._supported_tables: 1464 dfd.close() 1465 dfd = None 1466 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version))) 1467 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 1468 meta.decoder = codecs.getdecoder(sd) 1469 meta.encoder = codecs.getencoder(sd) 1470 fieldblock = dfd.read(header.start - 32) 1471 for i in range(len(fieldblock)//32+1): 1472 fieldend = i * 32 1473 if fieldblock[fieldend] == '\x0d': 1474 break 1475 else: 1476 raise DbfError("corrupt field structure in header") 1477 if len(fieldblock[:fieldend]) % 32 != 0: 1478 raise DbfError("corrupt field structure in header") 1479 header.fields = fieldblock[:fieldend] 1480 header.extra = fieldblock[fieldend+1:] # skip trailing \r 1481 yo._initializeFields() 1482 yo._checkMemoIntegrity() 1483 meta.current = -1 1484 if len(yo) > 0: 1485 meta.current = 0 1486 dfd.seek(0)
1487
1488 - def pack(yo, _pack=True):
1489 "physically removes all deleted records" 1490 for dbfindex in yo._indexen: 1491 dbfindex.clear() 1492 newtable = [] 1493 index = 0 1494 offset = 0 # +1 for each purged record 1495 for record in yo._table: 1496 found = False 1497 if record.has_been_deleted and _pack: 1498 for dbflist in yo._dbflists: 1499 if dbflist._purge(record, record.record_number - offset, 1): 1500 found = True 1501 record._recnum = -1 1502 else: 1503 record._recnum = index 1504 newtable.append(record) 1505 index += 1 1506 if found: 1507 offset += 1 1508 found = False 1509 yo._table.clear() 1510 for record in newtable: 1511 yo._table.append(record) 1512 yo._meta.header.record_count = index 1513 yo._current = -1 1514 yo._update_disk() 1515 yo.reindex()
1516 - def prev(yo):
1517 "set record pointer to previous (non-deleted) record, and return it" 1518 if yo.bof(_move=True): 1519 raise Bof 1520 return yo.current()
1521 - def query(yo, sql_command=None, python=None):
1522 "deprecated: use .find or .sql" 1523 if sql_command: 1524 return yo.sql(sql_command) 1525 elif python: 1526 return yo.find(python) 1527 raise DbfError("query: python parameter must be specified")
1528 - def reindex(yo):
1529 for dbfindex in yo._indexen: 1530 dbfindex.reindex()
1531 - def rename_field(yo, oldname, newname):
1532 "renames an existing field" 1533 if yo: 1534 yo.create_backup() 1535 if not oldname in yo._meta.fields: 1536 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname) 1537 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum(): 1538 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits") 1539 newname = newname.lower() 1540 if newname in yo._meta.fields: 1541 raise DbfError("field --%s-- already exists" % newname) 1542 if len(newname) > 10: 1543 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname))) 1544 yo._meta[newname] = yo._meta[oldname] 1545 yo._meta.fields[yo._meta.fields.index(oldname)] = newname 1546 yo._buildHeaderFields() 1547 yo._update_disk(headeronly=True)
1548 - def resize_field(yo, doomed, new_size):
1549 """resizes field (C only at this time) 1550 creates backup file, then modifies current structure""" 1551 if not 0 < new_size < 256: 1552 raise DbfError("new_size must be between 1 and 255 (use delete_fields to remove a field)") 1553 doomed = yo._list_fields(doomed) 1554 for victim in doomed: 1555 if victim not in yo._meta.fields: 1556 raise DbfError("field %s not in table -- resize aborted" % victim) 1557 all_records = [record for record in yo] 1558 yo.create_backup() 1559 for victim in doomed: 1560 start = yo._meta[victim]['start'] 1561 end = yo._meta[victim]['end'] 1562 eff_end = min(yo._meta[victim]['length'], new_size) 1563 yo._meta[victim]['length'] = new_size 1564 yo._meta[victim]['end'] = start + new_size 1565 blank = array('c', ' ' * new_size) 1566 for record in yo: 1567 new_data = blank[:] 1568 new_data[:eff_end] = record._data[start:start+eff_end] 1569 record._data = record._data[:start] + new_data + record._data[end:] 1570 for field in yo._meta.fields: 1571 if yo._meta[field]['start'] == end: 1572 end = yo._meta[field]['end'] 1573 yo._meta[field]['start'] = start + new_size 1574 yo._meta[field]['end'] = start + new_size + yo._meta[field]['length'] 1575 start = yo._meta[field]['end'] 1576 yo._buildHeaderFields() 1577 yo._update_disk()
1578 - def size(yo, field):
1579 "returns size of field as a tuple of (length, decimals)" 1580 if field in yo: 1581 return (yo._meta[field]['length'], yo._meta[field]['decimals']) 1582 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1583 - def sql(yo, command):
1584 "passes calls through to module level sql function" 1585 return sql(yo, command)
1586 - def structure(yo, fields=None):
1587 """return list of fields suitable for creating same table layout 1588 @param fields: list of fields or None for all fields""" 1589 field_specs = [] 1590 fields = yo._list_fields(fields) 1591 try: 1592 for name in fields: 1593 field_specs.append(yo._fieldLayout(yo.field_names.index(name))) 1594 except ValueError: 1595 raise DbfError("field --%s-- does not exist" % name) 1596 return field_specs
1597 - def top(yo, get_record=False):
1598 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record 1599 DbfError if table is empty 1600 Eof if all records are deleted and use_deleted is False""" 1601 yo._meta.current = -1 1602 if get_record: 1603 try: 1604 return yo.next() 1605 except Eof: 1606 yo._meta.current = -1 1607 raise Bof()
1608 - def type(yo, field):
1609 "returns type of field" 1610 if field in yo: 1611 return yo._meta[field]['type'] 1612 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1613 - def zap(yo, areyousure=False):
1614 """removes all records from table -- this cannot be undone! 1615 areyousure must be True, else error is raised""" 1616 if areyousure: 1617 if yo._meta.inmemory: 1618 yo._table = [] 1619 else: 1620 yo._table.clear() 1621 yo._meta.header.record_count = 0 1622 yo._current = -1 1623 yo._update_disk() 1624 else: 1625 raise DbfError("You must say you are sure to wipe the table")
1626 -class Db3Table(DbfTable):
1627 """Provides an interface for working with dBase III tables.""" 1628 _version = 'dBase III Plus' 1629 _versionabbv = 'db3' 1630 _fieldtypes = { 1631 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None}, 1632 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None}, 1633 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None}, 1634 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None}, 1635 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric, 'Class':None} } 1636 _memoext = '.dbt' 1637 _memotypes = ('M',) 1638 _memoClass = _Db3Memo 1639 _yesMemoMask = '\x80' 1640 _noMemoMask = '\x7f' 1641 _fixed_fields = ('D','L','M') 1642 _variable_fields = ('C','N') 1643 _character_fields = ('C','M') 1644 _decimal_fields = ('N',) 1645 _numeric_fields = ('N',) 1646 _currency_fields = tuple() 1647 _dbfTableHeader = array('c', '\x00' * 32) 1648 _dbfTableHeader[0] = '\x03' # version - dBase III w/o memo's 1649 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 1650 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1651 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1652 _dbfTableHeader = _dbfTableHeader.tostring() 1653 _dbfTableHeaderExtra = '' 1654 _supported_tables = ['\x03', '\x83'] 1655 _read_only = False 1656 _meta_only = False 1657 _use_deleted = True
1658 - def _checkMemoIntegrity(yo):
1659 "dBase III specific" 1660 if yo._meta.header.version == '\x83': 1661 try: 1662 yo._meta.memo = yo._memoClass(yo._meta) 1663 except: 1664 yo._meta.dfd.close() 1665 yo._meta.dfd = None 1666 raise 1667 if not yo._meta.ignorememos: 1668 for field in yo._meta.fields: 1669 if yo._meta[field]['type'] in yo._memotypes: 1670 if yo._meta.header.version != '\x83': 1671 yo._meta.dfd.close() 1672 yo._meta.dfd = None 1673 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 1674 elif not os.path.exists(yo._meta.memoname): 1675 yo._meta.dfd.close() 1676 yo._meta.dfd = None 1677 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1678 break
1679 - def _initializeFields(yo):
1680 "builds the FieldList of names, types, and descriptions" 1681 yo._meta.fields[:] = [] 1682 offset = 1 1683 fieldsdef = yo._meta.header.fields 1684 if len(fieldsdef) % 32 != 0: 1685 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 1686 if len(fieldsdef) // 32 != yo.field_count: 1687 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 1688 for i in range(yo.field_count): 1689 fieldblock = fieldsdef[i*32:(i+1)*32] 1690 name = io.unpackStr(fieldblock[:11]) 1691 type = fieldblock[11] 1692 if not type in yo._meta.fieldtypes: 1693 raise DbfError("Unknown field type: %s" % type) 1694 start = offset 1695 length = ord(fieldblock[16]) 1696 offset += length 1697 end = start + length 1698 decimals = ord(fieldblock[17]) 1699 flags = ord(fieldblock[18]) 1700 yo._meta.fields.append(name) 1701 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1702 -class FpTable(DbfTable):
1703 'Provides an interface for working with FoxPro 2 tables' 1704 _version = 'Foxpro' 1705 _versionabbv = 'fp' 1706 _fieldtypes = { 1707 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None}, 1708 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None}, 1709 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None}, 1710 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None}, 1711 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None}, 1712 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1713 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None}, 1714 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None}, 1715 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} } 1716 _memoext = '.fpt' 1717 _memotypes = ('G','M','P') 1718 _memoClass = _VfpMemo 1719 _yesMemoMask = '\xf5' # 1111 0101 1720 _noMemoMask = '\x03' # 0000 0011 1721 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 1722 _variable_fields = ('C','F','N') 1723 _character_fields = ('C','M') # field representing character data 1724 _decimal_fields = ('F','N') 1725 _numeric_fields = ('F','N') 1726 _currency_fields = tuple() 1727 _supported_tables = ('\x03', '\xf5') 1728 _dbfTableHeader = array('c', '\x00' * 32) 1729 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 1730 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263)) 1731 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1732 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1733 _dbfTableHeader = _dbfTableHeader.tostring() 1734 _dbfTableHeaderExtra = '\x00' * 263 1735 _use_deleted = True
1736 - def _checkMemoIntegrity(yo):
1737 if os.path.exists(yo._meta.memoname): 1738 try: 1739 yo._meta.memo = yo._memoClass(yo._meta) 1740 except: 1741 yo._meta.dfd.close() 1742 yo._meta.dfd = None 1743 raise 1744 if not yo._meta.ignorememos: 1745 for field in yo._meta.fields: 1746 if yo._meta[field]['type'] in yo._memotypes: 1747 if not os.path.exists(yo._meta.memoname): 1748 yo._meta.dfd.close() 1749 yo._meta.dfd = None 1750 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1751 break
1752 - def _initializeFields(yo):
1753 "builds the FieldList of names, types, and descriptions" 1754 yo._meta.fields[:] = [] 1755 offset = 1 1756 fieldsdef = yo._meta.header.fields 1757 if len(fieldsdef) % 32 != 0: 1758 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 1759 if len(fieldsdef) // 32 != yo.field_count: 1760 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 1761 for i in range(yo.field_count): 1762 fieldblock = fieldsdef[i*32:(i+1)*32] 1763 name = io.unpackStr(fieldblock[:11]) 1764 type = fieldblock[11] 1765 if not type in yo._meta.fieldtypes: 1766 raise DbfError("Unknown field type: %s" % type) 1767 elif type == '0': 1768 return # ignore nullflags 1769 start = offset 1770 length = ord(fieldblock[16]) 1771 offset += length 1772 end = start + length 1773 decimals = ord(fieldblock[17]) 1774 flags = ord(fieldblock[18]) 1775 yo._meta.fields.append(name) 1776 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1777
1778 -class VfpTable(DbfTable):
1779 'Provides an interface for working with Visual FoxPro 6 tables' 1780 _version = 'Visual Foxpro v6' 1781 _versionabbv = 'vfp' 1782 _fieldtypes = { 1783 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None}, 1784 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency, 'Class':None}, 1785 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble, 'Class':None}, 1786 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None}, 1787 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None}, 1788 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger, 'Class':None}, 1789 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None}, 1790 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None}, 1791 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime, 'Class':None}, 1792 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1793 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1794 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1795 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} } 1796 _memoext = '.fpt' 1797 _memotypes = ('G','M','P') 1798 _memoClass = _VfpMemo 1799 _yesMemoMask = '\x30' # 0011 0000 1800 _noMemoMask = '\x30' # 0011 0000 1801 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 1802 _variable_fields = ('C','F','N') 1803 _character_fields = ('C','M') # field representing character data 1804 _decimal_fields = ('F','N') 1805 _numeric_fields = ('B','F','I','N','Y') 1806 _currency_fields = ('Y',) 1807 _supported_tables = ('\x30',) 1808 _dbfTableHeader = array('c', '\x00' * 32) 1809 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 1810 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263)) 1811 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1812 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1813 _dbfTableHeader = _dbfTableHeader.tostring() 1814 _dbfTableHeaderExtra = '\x00' * 263 1815 _use_deleted = True
1816 - def _checkMemoIntegrity(yo):
1817 if os.path.exists(yo._meta.memoname): 1818 try: 1819 yo._meta.memo = yo._memoClass(yo._meta) 1820 except: 1821 yo._meta.dfd.close() 1822 yo._meta.dfd = None 1823 raise 1824 if not yo._meta.ignorememos: 1825 for field in yo._meta.fields: 1826 if yo._meta[field]['type'] in yo._memotypes: 1827 if not os.path.exists(yo._meta.memoname): 1828 yo._meta.dfd.close() 1829 yo._meta.dfd = None 1830 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1831 break
1832 - def _initializeFields(yo):
1833 "builds the FieldList of names, types, and descriptions" 1834 yo._meta.fields[:] = [] 1835 offset = 1 1836 fieldsdef = yo._meta.header.fields 1837 for i in range(yo.field_count): 1838 fieldblock = fieldsdef[i*32:(i+1)*32] 1839 name = io.unpackStr(fieldblock[:11]) 1840 type = fieldblock[11] 1841 if not type in yo._meta.fieldtypes: 1842 raise DbfError("Unknown field type: %s" % type) 1843 elif type == '0': 1844 return # ignore nullflags 1845 start = io.unpackLongInt(fieldblock[12:16]) 1846 length = ord(fieldblock[16]) 1847 offset += length 1848 end = start + length 1849 decimals = ord(fieldblock[17]) 1850 flags = ord(fieldblock[18]) 1851 yo._meta.fields.append(name) 1852 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1853 -class List(object):
1854 "list of Dbf records, with set-like behavior" 1855 _desc = ''
1856 - def __init__(yo, new_records=None, desc=None, key=None, field_names=None):
1857 yo.field_names = field_names 1858 yo._list = [] 1859 yo._set = set() 1860 if key is not None: 1861 yo.key = key 1862 if key.__doc__ is None: 1863 key.__doc__ = 'unknown' 1864 key = yo.key 1865 yo._current = -1 1866 if isinstance(new_records, yo.__class__) and key is new_records.key: 1867 yo._list = new_records._list[:] 1868 yo._set = new_records._set.copy() 1869 yo._current = 0 1870 elif new_records is not None: 1871 for record in new_records: 1872 value = key(record) 1873 item = (record.record_table, record.record_number, value) 1874 if value not in yo._set: 1875 yo._set.add(value) 1876 yo._list.append(item) 1877 yo._current = 0 1878 if desc is not None: 1879 yo._desc = desc
1880 - def __add__(yo, other):
1881 key = yo.key 1882 if isinstance(other, (DbfTable, list)): 1883 other = yo.__class__(other, key=key) 1884 if isinstance(other, yo.__class__): 1885 result = yo.__class__() 1886 result._set = yo._set.copy() 1887 result._list[:] = yo._list[:] 1888 result.key = yo.key 1889 if key is other.key: # same key? just compare key values 1890 for item in other._list: 1891 if item[2] not in result._set: 1892 result._set.add(item[2]) 1893 result._list.append(item) 1894 else: # different keys, use this list's key on other's records 1895 for rec in other: 1896 value = key(rec) 1897 if value not in result._set: 1898 result._set.add(value) 1899 result._list.append((rec.record_table, rec.record_number, value)) 1900 result._current = 0 if result else -1 1901 return result 1902 return NotImplemented
1903 - def __contains__(yo, record):
1904 if isinstance(record, tuple): 1905 item = record 1906 else: 1907 item = yo.key(record) 1908 return item in yo._set
1909 - def __delitem__(yo, key):
1910 if isinstance(key, int): 1911 item = yo._list.pop[key] 1912 yo._set.remove(item[2]) 1913 elif isinstance(key, slice): 1914 yo._set.difference_update([item[2] for item in yo._list[key]]) 1915 yo._list.__delitem__(key) 1916 elif isinstance(key, _DbfRecord): 1917 index = yo.index(key) 1918 item = yo._list.pop[index] 1919 yo._set.remove(item[2]) 1920 else: 1921 raise TypeError
1922 - def __getitem__(yo, key):
1923 if isinstance(key, int): 1924 count = len(yo._list) 1925 if not -count <= key < count: 1926 raise IndexError("Record %d is not in list." % key) 1927 return yo._get_record(*yo._list[key]) 1928 elif isinstance(key, slice): 1929 result = yo.__class__() 1930 result._list[:] = yo._list[key] 1931 result._set = set(result._list) 1932 result.key = yo.key 1933 result._current = 0 if result else -1 1934 return result 1935 elif isinstance(key, _DbfRecord): 1936 index = yo.index(key) 1937 return yo._get_record(*yo._list[index]) 1938 else: 1939 raise TypeError('indices must be integers')
1940 - def __iter__(yo):
1941 return (table.get_record(recno) for table, recno, value in yo._list)
1942 - def __len__(yo):
1943 return len(yo._list)
1944 - def __nonzero__(yo):
1945 return len(yo) > 0
1946 - def __radd__(yo, other):
1947 return yo.__add__(other)
1948 - def __repr__(yo):
1949 if yo._desc: 1950 return "%s(key=%s - %s - %d records)" % (yo.__class__, yo.key.__doc__, yo._desc, len(yo._list)) 1951 else: 1952 return "%s(key=%s - %d records)" % (yo.__class__, yo.key.__doc__, len(yo._list))
1953 - def __rsub__(yo, other):
1954 key = yo.key 1955 if isinstance(other, (DbfTable, list)): 1956 other = yo.__class__(other, key=key) 1957 if isinstance(other, yo.__class__): 1958 result = yo.__class__() 1959 result._list[:] = other._list[:] 1960 result._set = other._set.copy() 1961 result.key = key 1962 lost = set() 1963 if key is other.key: 1964 for item in yo._list: 1965 if item[2] in result._list: 1966 result._set.remove(item[2]) 1967 lost.add(item) 1968 else: 1969 for rec in other: 1970 value = key(rec) 1971 if value in result._set: 1972 result._set.remove(value) 1973 lost.add((rec.record_table, rec.record_number, value)) 1974 result._list = [item for item in result._list if item not in lost] 1975 result._current = 0 if result else -1 1976 return result 1977 return NotImplemented
1978 - def __sub__(yo, other):
1979 key = yo.key 1980 if isinstance(other, (DbfTable, list)): 1981 other = yo.__class__(other, key=key) 1982 if isinstance(other, yo.__class__): 1983 result = yo.__class__() 1984 result._list[:] = yo._list[:] 1985 result._set = yo._set.copy() 1986 result.key = key 1987 lost = set() 1988 if key is other.key: 1989 for item in other._list: 1990 if item[2] in result._set: 1991 result._set.remove(item[2]) 1992 lost.add(item[2]) 1993 else: 1994 for rec in other: 1995 value = key(rec) 1996 if value in result._set: 1997 result._set.remove(value) 1998 lost.add(value) 1999 result._list = [item for item in result._list if item[2] not in lost] 2000 result._current = 0 if result else -1 2001 return result 2002 return NotImplemented
2003 - def _maybe_add(yo, item):
2004 if item[2] not in yo._set: 2005 yo._set.add(item[2]) 2006 yo._list.append(item)
2007 - def _get_record(yo, table=None, rec_no=None, value=None):
2008 if table is rec_no is None: 2009 table, rec_no, value = yo._list[yo._current] 2010 return table.get_record(rec_no)
2011 - def _purge(yo, record, old_record_number, offset):
2012 partial = record.record_table, old_record_number 2013 records = sorted(yo._list, key=lambda item: (item[0], item[1])) 2014 for item in records: 2015 if partial == item[:2]: 2016 found = True 2017 break 2018 elif partial[0] is item[0] and partial[1] < item[1]: 2019 found = False 2020 break 2021 else: 2022 found = False 2023 if found: 2024 yo._list.pop(yo._list.index(item)) 2025 yo._set.remove(item[2]) 2026 start = records.index(item) + found 2027 for item in records[start:]: 2028 if item[0] is not partial[0]: # into other table's records 2029 break 2030 i = yo._list.index(item) 2031 yo._set.remove(item[2]) 2032 item = item[0], (item[1] - offset), item[2] 2033 yo._list[i] = item 2034 yo._set.add(item[2]) 2035 return found
2036 - def append(yo, new_record):
2037 yo._maybe_add((new_record.record_table, new_record.record_number, yo.key(new_record))) 2038 if yo._current == -1 and yo._list: 2039 yo._current = 0
2040 #return new_record
2041 - def bottom(yo):
2042 if yo._list: 2043 yo._current = len(yo._list) - 1 2044 return yo._get_record() 2045 raise DbfError("dbf.List is empty")
2046 - def clear(yo):
2047 yo._list = [] 2048 yo._set = set() 2049 yo._current = -1
2050 - def current(yo):
2051 if yo._current < 0: 2052 raise Bof() 2053 elif yo._current == len(yo._list): 2054 raise Eof() 2055 return yo._get_record()
2056 - def extend(yo, new_records):
2057 key = yo.key 2058 if isinstance(new_records, yo.__class__): 2059 if key is new_records.key: # same key? just compare key values 2060 for item in new_records._list: 2061 yo._maybe_add(item) 2062 else: # different keys, use this list's key on other's records 2063 for rec in new_records: 2064 value = key(rec) 2065 yo._maybe_add((rec.record_table, rec.record_number, value)) 2066 else: 2067 for record in new_records: 2068 value = key(rec) 2069 yo._maybe_add((rec.record_table, rec.record_number, value)) 2070 if yo._current == -1 and yo._list: 2071 yo._current = 0
2072 - def goto(yo, index_number):
2073 if yo._list: 2074 if 0 <= index_number <= len(yo._list): 2075 yo._current = index_number 2076 return yo._get_record() 2077 raise DbfError("index %d not in dbf.List of %d records" % (index_number, len(yo._list))) 2078 raise DbfError("dbf.List is empty")
2079 - def index(yo, sort=None, reverse=False):
2080 "sort= ((field_name, func), (field_name, func),) | 'ORIGINAL'" 2081 if sort is None: 2082 results = [] 2083 for field, func in yo._meta.index: 2084 results.append("%s(%s)" % (func.__name__, field)) 2085 return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed]) 2086 yo._meta.index_reversed = reverse 2087 if sort == 'ORIGINAL': 2088 yo._index = range(yo._meta.header.record_count) 2089 yo._meta.index = 'ORIGINAL' 2090 if reverse: 2091 yo._index.reverse() 2092 return 2093 new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop]) 2094 yo._meta.index = tuple(new_sort) 2095 yo._meta.orderresults = [''] * len(yo) 2096 for record in yo: 2097 yo._meta.orderresults[record.record_number] = record() 2098 yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
2099 - def index(yo, record, start=None, stop=None):
2100 item = record.record_table, record.record_number, yo.key(record) 2101 key = yo.key(record) 2102 if start is None: 2103 start = 0 2104 if stop is None: 2105 stop = len(yo._list) 2106 for i in range(start, stop): 2107 if yo._list[i][2] == key: 2108 return i 2109 else: 2110 raise ValueError("dbf.List.index(x): <x=%r> not in list" % (key,))
2111 - def insert(yo, i, record):
2112 item = record.record_table, record.record_number, yo.key(record) 2113 if item not in yo._set: 2114 yo._set.add(item[2]) 2115 yo._list.insert(i, item)
2116 - def key(yo, record):
2117 "table_name, record_number" 2118 return record.record_table, record.record_number
2119 - def next(yo):
2120 if yo._current < len(yo._list): 2121 yo._current += 1 2122 if yo._current < len(yo._list): 2123 return yo._get_record() 2124 raise Eof()
2125 - def pop(yo, index=None):
2126 if index is None: 2127 table, recno, value = yo._list.pop() 2128 else: 2129 table, recno, value = yo._list.pop(index) 2130 yo._set.remove(value) 2131 return yo._get_record(table, recno, value)
2132 - def prev(yo):
2133 if yo._current >= 0: 2134 yo._current -= 1 2135 if yo._current > -1: 2136 return yo._get_record() 2137 raise Bof()
2138 - def remove(yo, record):
2139 item = record.record_table, record.record_number, yo.key(record) 2140 yo._list.remove(item) 2141 yo._set.remove(item[2])
2142 - def reverse(yo):
2143 return yo._list.reverse()
2144 - def top(yo):
2145 if yo._list: 2146 yo._current = 0 2147 return yo._get_record() 2148 raise DbfError("dbf.List is empty")
2149 - def sort(yo, key=None, reverse=False):
2150 if key is None: 2151 return yo._list.sort(reverse=reverse) 2152 return yo._list.sort(key=lambda item: key(item[0].get_record(item[1])), reverse=reverse)
2153
2154 -class DbfCsv(csv.Dialect):
2155 "csv format for exporting tables" 2156 delimiter = ',' 2157 doublequote = True 2158 escapechar = None 2159 lineterminator = '\n' 2160 quotechar = '"' 2161 skipinitialspace = True 2162 quoting = csv.QUOTE_NONNUMERIC
2163 -class Index(object):
2164 - class IndexIterator(object):
2165 "returns records using this index"
2166 - def __init__(yo, table, records):
2167 yo.table = table 2168 yo.records = records 2169 yo.index = 0
2170 - def __iter__(yo):
2171 return yo
2172 - def next(yo):
2173 while yo.index < len(yo.records): 2174 record = yo.table.get_record(yo.records[yo.index]) 2175 yo.index += 1 2176 if not yo.table.use_deleted and record.has_been_deleted: 2177 continue 2178 return record 2179 else: 2180 raise StopIteration
2181 - def __init__(yo, table, key, field_names=None):
2182 yo._table = table 2183 yo._values = [] # ordered list of values 2184 yo._rec_by_val = [] # matching record numbers 2185 yo._records = {} # record numbers:values 2186 yo.__doc__ = key.__doc__ or 'unknown' 2187 yo.key = key 2188 yo.field_names = field_names or table.field_names 2189 for record in table: 2190 value = key(record) 2191 if value is DoNotIndex: 2192 continue 2193 rec_num = record.record_number 2194 if not isinstance(value, tuple): 2195 value = (value, ) 2196 vindex = bisect_right(yo._values, value) 2197 yo._values.insert(vindex, value) 2198 yo._rec_by_val.insert(vindex, rec_num) 2199 yo._records[rec_num] = value 2200 table._indexen.add(yo)
2201 - def __call__(yo, record):
2202 rec_num = record.record_number 2203 if rec_num in yo._records: 2204 value = yo._records[rec_num] 2205 vindex = bisect_left(yo._values, value) 2206 yo._values.pop(vindex) 2207 yo._rec_by_val.pop(vindex) 2208 value = yo.key(record) 2209 if value is DoNotIndex: 2210 return 2211 if not isinstance(value, tuple): 2212 value = (value, ) 2213 vindex = bisect_right(yo._values, value) 2214 yo._values.insert(vindex, value) 2215 yo._rec_by_val.insert(vindex, rec_num) 2216 yo._records[rec_num] = value
2217 - def __contains__(yo, match):
2218 if isinstance(match, _DbfRecord): 2219 if match.record_table is yo._table: 2220 return match.record_number in yo._records 2221 match = yo.key(match) 2222 elif not isinstance(match, tuple): 2223 match = (match, ) 2224 return yo.find(match) != -1
2225 - def __getitem__(yo, key):
2226 if isinstance(key, int): 2227 count = len(yo._values) 2228 if not -count <= key < count: 2229 raise IndexError("Record %d is not in list." % key) 2230 rec_num = yo._rec_by_val[key] 2231 return yo._table.get_record(rec_num) 2232 elif isinstance(key, slice): 2233 result = List(field_names=yo._table.field_names) 2234 yo._table._dbflists.add(result) 2235 start, stop, step = key.start, key.stop, key.step 2236 if start is None: start = 0 2237 if stop is None: stop = len(yo._rec_by_val) 2238 if step is None: step = 1 2239 for loc in range(start, stop, step): 2240 record = yo._table.get_record(yo._rec_by_val[loc]) 2241 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2242 result._current = 0 if result else -1 2243 return result 2244 elif isinstance (key, (str, unicode, tuple, _DbfRecord)): 2245 if isinstance(key, _DbfRecord): 2246 key = yo.key(key) 2247 elif not isinstance(key, tuple): 2248 key = (key, ) 2249 loc = yo.find(key) 2250 if loc == -1: 2251 raise KeyError(key) 2252 return yo._table.get_record(yo._rec_by_val[loc]) 2253 else: 2254 raise TypeError('indices must be integers, match objects must by strings or tuples')
2255 - def __enter__(yo):
2256 return yo
2257 - def __exit__(yo, *exc_info):
2258 yo._table.close() 2259 yo._values[:] = [] 2260 yo._rec_by_val[:] = [] 2261 yo._records.clear() 2262 return False
2263 - def __iter__(yo):
2264 return yo.IndexIterator(yo._table, yo._rec_by_val)
2265 - def __len__(yo):
2266 return len(yo._records)
2267 - def _partial_match(yo, target, match):
2268 target = target[:len(match)] 2269 if isinstance(match[-1], (str, unicode)): 2270 target = list(target) 2271 target[-1] = target[-1][:len(match[-1])] 2272 target = tuple(target) 2273 return target == match
2274 - def _purge(yo, rec_num):
2275 value = yo._records.get(rec_num) 2276 if value is not None: 2277 vindex = bisect_left(yo._values, value) 2278 del yo._records[rec_num] 2279 yo._values.pop(vindex) 2280 yo._rec_by_val.pop(vindex)
2281 - def _search(yo, match, lo=0, hi=None):
2282 if hi is None: 2283 hi = len(yo._values) 2284 return bisect_left(yo._values, match, lo, hi)
2285 - def clear(yo):
2286 "removes all entries from index" 2287 yo._values[:] = [] 2288 yo._rec_by_val[:] = [] 2289 yo._records.clear()
2290 - def close(yo):
2291 yo._table.close()
2292 - def find(yo, match, partial=False):
2293 "returns numeric index of (partial) match, or -1" 2294 if isinstance(match, _DbfRecord): 2295 if match.record_number in yo._records: 2296 return yo._values.index(yo._records[match.record_number]) 2297 else: 2298 return -1 2299 if not isinstance(match, tuple): 2300 match = (match, ) 2301 loc = yo._search(match) 2302 while loc < len(yo._values) and yo._values[loc] == match: 2303 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted: 2304 loc += 1 2305 continue 2306 return loc 2307 if partial: 2308 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match): 2309 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted: 2310 loc += 1 2311 continue 2312 return loc 2313 return -1
2314 - def find_index(yo, match):
2315 "returns numeric index of either (partial) match, or position of where match would be" 2316 if isinstance(match, _DbfRecord): 2317 if match.record_number in yo._records: 2318 return yo._values.index(yo._records[match.record_number]) 2319 else: 2320 match = yo.key(match) 2321 if not isinstance(match, tuple): 2322 match = (match, ) 2323 loc = yo._search(match) 2324 return loc
2325 - def index(yo, match, partial=False):
2326 "returns numeric index of (partial) match, or raises ValueError" 2327 loc = yo.find(match, partial) 2328 if loc == -1: 2329 if isinstance(match, _DbfRecord): 2330 raise ValueError("table <%s> record [%d] not in index <%s>" % (yo._table.filename, match.record_number, yo.__doc__)) 2331 else: 2332 raise ValueError("match criteria <%s> not in index" % (match, )) 2333 return loc
2334 - def reindex(yo):
2335 "reindexes all records" 2336 for record in yo._table: 2337 yo(record)
2338 - def query(yo, sql_command=None, python=None):
2339 """recognized sql commands are SELECT, UPDATE, REPLACE, INSERT, DELETE, and RECALL""" 2340 if sql_command: 2341 return sql(yo, sql_command) 2342 elif python is None: 2343 raise DbfError("query: python parameter must be specified") 2344 possible = List(desc="%s --> %s" % (yo._table.filename, python), field_names=yo._table.field_names) 2345 yo._table._dbflists.add(possible) 2346 query_result = {} 2347 select = 'query_result["keep"] = %s' % python 2348 g = {} 2349 for record in yo: 2350 query_result['keep'] = False 2351 g['query_result'] = query_result 2352 exec select in g, record 2353 if query_result['keep']: 2354 possible.append(record) 2355 record.write_record() 2356 return possible
2357 - def search(yo, match, partial=False):
2358 "returns dbf.List of all (partially) matching records" 2359 result = List(field_names=yo._table.field_names) 2360 yo._table._dbflists.add(result) 2361 if not isinstance(match, tuple): 2362 match = (match, ) 2363 loc = yo._search(match) 2364 if loc == len(yo._values): 2365 return result 2366 while loc < len(yo._values) and yo._values[loc] == match: 2367 record = yo._table.get_record(yo._rec_by_val[loc]) 2368 if not yo._table.use_deleted and record.has_been_deleted: 2369 loc += 1 2370 continue 2371 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2372 loc += 1 2373 if partial: 2374 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match): 2375 record = yo._table.get_record(yo._rec_by_val[loc]) 2376 if not yo._table.use_deleted and record.has_been_deleted: 2377 loc += 1 2378 continue 2379 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2380 loc += 1 2381 return result
2382 2383 csv.register_dialect('dbf', DbfCsv)
2384 2385 -def sql_select(records, chosen_fields, condition, field_names):
2386 if chosen_fields != '*': 2387 field_names = chosen_fields.replace(' ','').split(',') 2388 result = condition(records) 2389 result.modified = 0, 'record' + ('','s')[len(result)>1] 2390 result.field_names = field_names 2391 return result
2392
2393 -def sql_update(records, command, condition, field_names):
2394 possible = condition(records) 2395 modified = sql_cmd(command, field_names)(possible) 2396 possible.modified = modified, 'record' + ('','s')[modified>1] 2397 return possible
2398
2399 -def sql_delete(records, dead_fields, condition, field_names):
2400 deleted = condition(records) 2401 deleted.modified = len(deleted), 'record' + ('','s')[len(deleted)>1] 2402 deleted.field_names = field_names 2403 if dead_fields == '*': 2404 for record in deleted: 2405 record.delete_record() 2406 record.write_record() 2407 else: 2408 keep = [f for f in field_names if f not in dead_fields.replace(' ','').split(',')] 2409 for record in deleted: 2410 record.reset_record(keep_fields=keep) 2411 record.write_record() 2412 return deleted
2413
2414 -def sql_recall(records, all_fields, condition, field_names):
2415 if all_fields != '*': 2416 raise DbfError('SQL RECALL: fields must be * (only able to recover at the record level)') 2417 revivified = List() 2418 tables = set() 2419 for record in records: 2420 tables.add(record_table) 2421 old_setting = dict() 2422 for table in tables: 2423 old_setting[table] = table.use_deleted 2424 table.use_deleted = True 2425 for record in condition(records): 2426 if record.has_been_deleted: 2427 revivified.append(record) 2428 record.undelete_record() 2429 record.write_record() 2430 for table in tables: 2431 table.use_deleted = old_setting[table] 2432 revivified.modfied = len(revivified), 'record' + ('','s')[len(revivified)>1] 2433 revivified.field_names = field_names 2434 return revivified
2435
2436 -def sql_add(records, new_fields, condition, field_names):
2437 tables = set() 2438 possible = condition(records) 2439 for record in possible: 2440 tables.add(record.record_table) 2441 for table in tables: 2442 table.add_fields(new_fields) 2443 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2444 possible.field_names = field_names 2445 return possible
2446
2447 -def sql_drop(records, dead_fields, condition, field_names):
2448 tables = set() 2449 possible = condition(records) 2450 for record in possible: 2451 tables.add(record.record_table) 2452 for table in tables: 2453 table.delete_fields(dead_fields) 2454 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2455 possible.field_names = field_names 2456 return possible
2457
2458 -def sql_pack(records, command, condition, field_names):
2459 tables = set() 2460 possible = condition(records) 2461 for record in possible: 2462 tables.add(record.record_table) 2463 for table in tables: 2464 table.pack() 2465 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2466 possible.field_names = field_names 2467 return possible
2468
2469 -def sql_resize(records, fieldname_newsize, condition, field_names):
2470 tables = set() 2471 possible = condition(records) 2472 for record in possible: 2473 tables.add(record.record_table) 2474 fieldname, newsize = fieldname_newsize.split() 2475 newsize = int(newsize) 2476 for table in tables: 2477 table.resize_field(fieldname, newsize) 2478 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2479 possible.field_names = field_names 2480 return possible
2481 2482 sql_functions = { 2483 'select' : sql_select, 2484 'update' : sql_update, 2485 'replace': sql_update, 2486 'insert' : None, 2487 'delete' : sql_delete, 2488 'recall' : sql_recall, 2489 'add' : sql_add, 2490 'drop' : sql_drop, 2491 'count' : None, 2492 'pack' : sql_pack, 2493 'resize' : sql_resize, 2494 }
2495 2496 -def sql_criteria(records, criteria):
2497 "creates a function matching the sql criteria" 2498 function = """def func(records): 2499 \"\"\"%s\"\"\" 2500 matched = List(field_names=records[0].field_names) 2501 for rec in records: 2502 %s 2503 2504 if %s: 2505 matched.append(rec) 2506 return matched""" 2507 fields = [] 2508 for field in records[0].field_names: 2509 if field in criteria: 2510 fields.append(field) 2511 fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields]) 2512 g = dbf.sql_user_functions.copy() 2513 g['List'] = List 2514 function %= (criteria, fields, criteria) 2515 #print function 2516 exec function in g 2517 return g['func']
2518
2519 -def sql_cmd(command, field_names):
2520 "creates a function matching to apply command to each record in records" 2521 function = """def func(records): 2522 \"\"\"%s\"\"\" 2523 changed = 0 2524 for rec in records: 2525 %s 2526 2527 %s 2528 2529 %s 2530 changed += rec.write_record() 2531 return changed""" 2532 fields = [] 2533 for field in field_names: 2534 if field in command: 2535 fields.append(field) 2536 pre_fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields]) 2537 post_fields = '\n '.join(['rec.%s = %s' % (field, field) for field in fields]) 2538 g = dbf.sql_user_functions.copy() 2539 if '=' not in command and ' with ' in command.lower(): 2540 offset = command.lower().index(' with ') 2541 command = command[:offset] + ' = ' + command[offset+6:] 2542 function %= (command, pre_fields, command, post_fields) 2543 #print function 2544 exec function in g 2545 return g['func']
2546
2547 -def sql(records, command):
2548 """recognized sql commands are SELECT, UPDATE | REPLACE, DELETE, RECALL, ADD, DROP""" 2549 sql_command = command 2550 if ' where ' in command: 2551 command, condition = command.split(' where ', 1) 2552 condition = sql_criteria(records, condition) 2553 else: 2554 def condition(records): 2555 return records[:]
2556 name, command = command.split(' ', 1) 2557 command = command.strip() 2558 name = name.lower() 2559 field_names = records[0].field_names 2560 if sql_functions.get(name) is None: 2561 raise DbfError('unknown SQL command: %s' % name.upper()) 2562 result = sql_functions[name](records, command, condition, field_names) 2563 tables = set() 2564 for record in result: 2565 tables.add(record.record_table) 2566 for list_table in tables: 2567 list_table._dbflists.add(result) 2568 return result
2569 -def _nop(value):
2570 "returns parameter unchanged" 2571 return value
2572 -def _normalize_tuples(tuples, length, filler):
2573 "ensures each tuple is the same length, using filler[-missing] for the gaps" 2574 final = [] 2575 for t in tuples: 2576 if len(t) < length: 2577 final.append( tuple([item for item in t] + filler[len(t)-length:]) ) 2578 else: 2579 final.append(t) 2580 return tuple(final)
2581 -def _codepage_lookup(cp):
2582 if cp not in code_pages: 2583 for code_page in sorted(code_pages.keys()): 2584 sd, ld = code_pages[code_page] 2585 if cp == sd or cp == ld: 2586 if sd is None: 2587 raise DbfError("Unsupported codepage: %s" % ld) 2588 cp = code_page 2589 break 2590 else: 2591 raise DbfError("Unsupported codepage: %s" % cp) 2592 sd, ld = code_pages[cp] 2593 return cp, sd, ld
2594 -def ascii(new_setting=None):
2595 "get/set return_ascii setting" 2596 global return_ascii 2597 if new_setting is None: 2598 return return_ascii 2599 else: 2600 return_ascii = new_setting
2601 -def codepage(cp=None):
2602 "get/set default codepage for any new tables" 2603 global default_codepage 2604 cp, sd, ld = _codepage_lookup(cp or default_codepage) 2605 default_codepage = sd 2606 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2607 -def encoding(cp=None):
2608 "get/set default encoding for non-unicode strings passed into a table" 2609 global input_decoding 2610 cp, sd, ld = _codepage_lookup(cp or input_decoding) 2611 default_codepage = sd 2612 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2613 -class _Db4Table(DbfTable):
2614 version = 'dBase IV w/memos (non-functional)' 2615 _versionabbv = 'db4' 2616 _fieldtypes = { 2617 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 2618 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency}, 2619 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble}, 2620 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric}, 2621 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric}, 2622 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger}, 2623 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 2624 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 2625 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime}, 2626 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2627 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2628 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2629 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} } 2630 _memoext = '.dbt' 2631 _memotypes = ('G','M','P') 2632 _memoClass = _VfpMemo 2633 _yesMemoMask = '\x8b' # 0011 0000 2634 _noMemoMask = '\x04' # 0011 0000 2635 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 2636 _variable_fields = ('C','F','N') 2637 _character_fields = ('C','M') # field representing character data 2638 _decimal_fields = ('F','N') 2639 _numeric_fields = ('B','F','I','N','Y') 2640 _currency_fields = ('Y',) 2641 _supported_tables = ('\x04', '\x8b') 2642 _dbfTableHeader = ['\x00'] * 32 2643 _dbfTableHeader[0] = '\x8b' # version - Foxpro 6 0011 0000 2644 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 2645 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 2646 _dbfTableHeader = ''.join(_dbfTableHeader) 2647 _dbfTableHeaderExtra = '' 2648 _use_deleted = True
2649 - def _checkMemoIntegrity(yo):
2650 "dBase III specific" 2651 if yo._meta.header.version == '\x8b': 2652 try: 2653 yo._meta.memo = yo._memoClass(yo._meta) 2654 except: 2655 yo._meta.dfd.close() 2656 yo._meta.dfd = None 2657 raise 2658 if not yo._meta.ignorememos: 2659 for field in yo._meta.fields: 2660 if yo._meta[field]['type'] in yo._memotypes: 2661 if yo._meta.header.version != '\x8b': 2662 yo._meta.dfd.close() 2663 yo._meta.dfd = None 2664 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 2665 elif not os.path.exists(yo._meta.memoname): 2666 yo._meta.dfd.close() 2667 yo._meta.dfd = None 2668 raise DbfError("Table structure corrupt: memo fields exist without memo file") 2669 break
2670