1 "table definitions"
2 import os
3 import sys
4 import csv
5 import codecs
6 import locale
7 import unicodedata
8 import weakref
9 from array import array
10 from bisect import bisect_left, bisect_right
11 from decimal import Decimal
12 from shutil import copyfileobj
13 import dbf
14 from dbf import _io as io
15 from dbf.dates import Date, DateTime, Time
16 from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode, DoNotIndex
17
18 input_decoding = locale.getdefaultlocale()[1]
19 default_codepage = 'cp1252'
20 return_ascii = False
21
22 version_map = {
23 '\x02' : 'FoxBASE',
24 '\x03' : 'dBase III Plus',
25 '\x04' : 'dBase IV',
26 '\x05' : 'dBase V',
27 '\x30' : 'Visual FoxPro',
28 '\x31' : 'Visual FoxPro (auto increment field)',
29 '\x43' : 'dBase IV SQL',
30 '\x7b' : 'dBase IV w/memos',
31 '\x83' : 'dBase III Plus w/memos',
32 '\x8b' : 'dBase IV w/memos',
33 '\x8e' : 'dBase IV w/SQL table',
34 '\xf5' : 'FoxPro w/memos'}
35
36 code_pages = {
37 '\x00' : ('ascii', "plain ol' ascii"),
38 '\x01' : ('cp437', 'U.S. MS-DOS'),
39 '\x02' : ('cp850', 'International MS-DOS'),
40 '\x03' : ('cp1252', 'Windows ANSI'),
41 '\x04' : ('mac_roman', 'Standard Macintosh'),
42 '\x08' : ('cp865', 'Danish OEM'),
43 '\x09' : ('cp437', 'Dutch OEM'),
44 '\x0A' : ('cp850', 'Dutch OEM (secondary)'),
45 '\x0B' : ('cp437', 'Finnish OEM'),
46 '\x0D' : ('cp437', 'French OEM'),
47 '\x0E' : ('cp850', 'French OEM (secondary)'),
48 '\x0F' : ('cp437', 'German OEM'),
49 '\x10' : ('cp850', 'German OEM (secondary)'),
50 '\x11' : ('cp437', 'Italian OEM'),
51 '\x12' : ('cp850', 'Italian OEM (secondary)'),
52 '\x13' : ('cp932', 'Japanese Shift-JIS'),
53 '\x14' : ('cp850', 'Spanish OEM (secondary)'),
54 '\x15' : ('cp437', 'Swedish OEM'),
55 '\x16' : ('cp850', 'Swedish OEM (secondary)'),
56 '\x17' : ('cp865', 'Norwegian OEM'),
57 '\x18' : ('cp437', 'Spanish OEM'),
58 '\x19' : ('cp437', 'English OEM (Britain)'),
59 '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'),
60 '\x1B' : ('cp437', 'English OEM (U.S.)'),
61 '\x1C' : ('cp863', 'French OEM (Canada)'),
62 '\x1D' : ('cp850', 'French OEM (secondary)'),
63 '\x1F' : ('cp852', 'Czech OEM'),
64 '\x22' : ('cp852', 'Hungarian OEM'),
65 '\x23' : ('cp852', 'Polish OEM'),
66 '\x24' : ('cp860', 'Portugese OEM'),
67 '\x25' : ('cp850', 'Potugese OEM (secondary)'),
68 '\x26' : ('cp866', 'Russian OEM'),
69 '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'),
70 '\x40' : ('cp852', 'Romanian OEM'),
71 '\x4D' : ('cp936', 'Chinese GBK (PRC)'),
72 '\x4E' : ('cp949', 'Korean (ANSI/OEM)'),
73 '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'),
74 '\x50' : ('cp874', 'Thai (ANSI/OEM)'),
75 '\x57' : ('cp1252', 'ANSI'),
76 '\x58' : ('cp1252', 'Western European ANSI'),
77 '\x59' : ('cp1252', 'Spanish ANSI'),
78 '\x64' : ('cp852', 'Eastern European MS-DOS'),
79 '\x65' : ('cp866', 'Russian MS-DOS'),
80 '\x66' : ('cp865', 'Nordic MS-DOS'),
81 '\x67' : ('cp861', 'Icelandic MS-DOS'),
82 '\x68' : (None, 'Kamenicky (Czech) MS-DOS'),
83 '\x69' : (None, 'Mazovia (Polish) MS-DOS'),
84 '\x6a' : ('cp737', 'Greek MS-DOS (437G)'),
85 '\x6b' : ('cp857', 'Turkish MS-DOS'),
86 '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'),
87 '\x79' : ('cp949', 'Korean Windows'),
88 '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'),
89 '\x7b' : ('cp932', 'Japanese Windows'),
90 '\x7c' : ('cp874', 'Thai Windows'),
91 '\x7d' : ('cp1255', 'Hebrew Windows'),
92 '\x7e' : ('cp1256', 'Arabic Windows'),
93 '\xc8' : ('cp1250', 'Eastern European Windows'),
94 '\xc9' : ('cp1251', 'Russian Windows'),
95 '\xca' : ('cp1254', 'Turkish Windows'),
96 '\xcb' : ('cp1253', 'Greek Windows'),
97 '\x96' : ('mac_cyrillic', 'Russian Macintosh'),
98 '\x97' : ('mac_latin2', 'Macintosh EE'),
99 '\x98' : ('mac_greek', 'Greek Macintosh') }
100
101 if sys.version_info[:2] < (2, 6):
104 "Emulate PyProperty_Type() in Objects/descrobject.c"
105
106 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
107 self.fget = fget
108 self.fset = fset
109 self.fdel = fdel
110 self.__doc__ = doc or fget.__doc__
112 self.fget = func
113 if not self.__doc__:
114 self.__doc__ = fget.__doc__
115 - def __get__(self, obj, objtype=None):
116 if obj is None:
117 return self
118 if self.fget is None:
119 raise AttributeError, "unreadable attribute"
120 return self.fget(obj)
122 if self.fset is None:
123 raise AttributeError, "can't set attribute"
124 self.fset(obj, value)
126 if self.fdel is None:
127 raise AttributeError, "can't delete attribute"
128 self.fdel(obj)
130 self.fset = func
131 return self
133 self.fdel = func
134 return self
135
137 """Provides routines to extract and save data within the fields of a dbf record."""
138 __slots__ = ['_recnum', '_layout', '_data', '_dirty', '__weakref__']
140 """calls appropriate routine to fetch value stored in field from array
141 @param record_data: the data portion of the record
142 @type record_data: array of characters
143 @param fielddef: description of the field definition
144 @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags'
145 @returns: python data stored in field"""
146
147 field_type = fielddef['type']
148 retrieve = yo._layout.fieldtypes[field_type]['Retrieve']
149 datum = retrieve(record_data, fielddef, yo._layout.memo)
150 if field_type in yo._layout.character_fields:
151 datum = yo._layout.decoder(datum)[0]
152 if yo._layout.return_ascii:
153 try:
154 datum = yo._layout.output_encoder(datum)[0]
155 except UnicodeEncodeError:
156 datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore')
157 return datum
159 "calls appropriate routine to convert value to ascii bytes, and save it in record"
160 field_type = fielddef['type']
161 update = yo._layout.fieldtypes[field_type]['Update']
162 if field_type in yo._layout.character_fields:
163 if not isinstance(value, unicode):
164 if yo._layout.input_decoder is None:
165 raise NonUnicode("String not in unicode format, no default encoding specified")
166 value = yo._layout.input_decoder(value)[0]
167 value = yo._layout.encoder(value)[0]
168 bytes = array('c', update(value, fielddef, yo._layout.memo))
169 size = fielddef['length']
170 if len(bytes) > size:
171 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size))
172 blank = array('c', ' ' * size)
173 start = fielddef['start']
174 end = start + size
175 blank[:len(bytes)] = bytes[:]
176 yo._data[start:end] = blank[:]
177 yo._dirty = True
192 results = []
193 if not specs:
194 specs = yo._layout.index
195 specs = _normalize_tuples(tuples=specs, length=2, filler=[_nop])
196 for field, func in specs:
197 results.append(func(yo[field]))
198 return tuple(results)
199
205 if name[0:2] == '__' and name[-2:] == '__':
206 raise AttributeError, 'Method %s is not implemented.' % name
207 elif name == 'record_number':
208 return yo._recnum
209 elif name == 'delete_flag':
210 return yo._data[0] != ' '
211 elif not name in yo._layout.fields:
212 raise FieldMissing(name)
213 try:
214 fielddef = yo._layout[name]
215 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef)
216 return value
217 except DbfError, error:
218 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message)
219 raise
236 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
275 if type(name) == str:
276 yo.__setattr__(name, value)
277 elif type(name) in (int, long):
278 yo.__setattr__(yo._layout.fields[name], value)
279 elif type(name) == slice:
280 sequence = []
281 for field in yo._layout.fields[name]:
282 sequence.append(field)
283 if len(sequence) != len(value):
284 raise DbfError("length of slices not equal")
285 for field, val in zip(sequence, value):
286 yo[field] = val
287 else:
288 raise TypeError("%s is not a field name" % name)
290 result = []
291 for seq, field in enumerate(yo.field_names):
292 result.append("%3d - %-10s: %s" % (seq, field, yo[field]))
293 return '\n'.join(result)
295 return yo._data.tostring()
297 "creates a blank record data chunk"
298 layout = yo._layout
299 ondisk = layout.ondisk
300 layout.ondisk = False
301 yo._data = array('c', ' ' * layout.header.record_length)
302 layout.memofields = []
303 for field in layout.fields:
304 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']())
305 if layout[field]['type'] in layout.memotypes:
306 layout.memofields.append(field)
307 layout.blankrecord = yo._data[:]
308 layout.ondisk = ondisk
310 "marks record as deleted"
311 yo._data[0] = '*'
312 yo._dirty = True
313 return yo
314 @property
319 "saves a dictionary into a record's fields\nkeys with no matching field will raise a FieldMissing exception unless drop_missing = True"
320 old_data = yo._data[:]
321 try:
322 for key in dictionary:
323 if not key in yo.field_names:
324 if drop:
325 continue
326 raise FieldMissing(key)
327 yo.__setattr__(key, dictionary[key])
328 except:
329 yo._data[:] = old_data
330 raise
331 return yo
332 @property
334 "marked for deletion?"
335 return yo._data[0] == '*'
344 @property
346 "physical record number"
347 return yo._recnum
348 @property
350 table = yo._layout.table()
351 if table is None:
352 raise DbfError("table is no longer available")
353 return table
355 for dbfindex in yo._layout.table()._indexen:
356 dbfindex(yo)
358 "blanks record"
359 if keep_fields is None:
360 keep_fields = []
361 keep = {}
362 for field in keep_fields:
363 keep[field] = yo[field]
364 if yo._layout.blankrecord == None:
365 yo._createBlankRecord()
366 yo._data[:] = yo._layout.blankrecord[:]
367 for field in keep_fields:
368 yo[field] = keep[field]
369 yo._dirty = True
370 return yo
372 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty."
373 keys = yo._layout.fields
374 if blank:
375 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys]
376 else:
377 values = [yo[field] for field in keys]
378 return dict(zip(keys, values))
380 "marks record as active"
381 yo._data[0] = ' '
382 yo._dirty = True
383 return yo
393 """Provides access to memo fields as dictionaries
394 must override _init, _get_memo, and _put_memo to
395 store memo contents to disk"""
397 "initialize disk file usage"
399 "retrieve memo contents from disk"
401 "store memo contents to disk"
403 ""
404 yo.meta = meta
405 yo.memory = {}
406 yo.nextmemo = 1
407 yo._init()
408 yo.meta.newmemofile = False
410 "gets the memo in block"
411 if yo.meta.ignorememos or not block:
412 return ''
413 if yo.meta.ondisk:
414 return yo._get_memo(block)
415 else:
416 return yo.memory[block]
418 "stores data in memo file, returns block number"
419 if yo.meta.ignorememos or data == '':
420 return 0
421 if yo.meta.inmemory:
422 thismemo = yo.nextmemo
423 yo.nextmemo += 1
424 yo.memory[thismemo] = data
425 else:
426 thismemo = yo._put_memo(data)
427 return thismemo
445 block = int(block)
446 yo.meta.mfd.seek(block * yo.meta.memo_size)
447 eom = -1
448 data = ''
449 while eom == -1:
450 newdata = yo.meta.mfd.read(yo.meta.memo_size)
451 if not newdata:
452 return data
453 data += newdata
454 eom = data.find('\x1a\x1a')
455 return data[:eom].rstrip()
457 data = data.rstrip()
458 length = len(data) + yo.record_header_length
459 blocks = length // yo.meta.memo_size
460 if length % yo.meta.memo_size:
461 blocks += 1
462 thismemo = yo.nextmemo
463 yo.nextmemo = thismemo + blocks
464 yo.meta.mfd.seek(0)
465 yo.meta.mfd.write(io.packLongInt(yo.nextmemo))
466 yo.meta.mfd.seek(thismemo * yo.meta.memo_size)
467 yo.meta.mfd.write(data)
468 yo.meta.mfd.write('\x1a\x1a')
469 double_check = yo._get_memo(thismemo)
470 if len(double_check) != len(data):
471 uhoh = open('dbf_memo_dump.err','wb')
472 uhoh.write('thismemo: %d' % thismemo)
473 uhoh.write('nextmemo: %d' % yo.nextmemo)
474 uhoh.write('saved: %d bytes' % len(data))
475 uhoh.write(data)
476 uhoh.write('retrieved: %d bytes' % len(double_check))
477 uhoh.write(double_check)
478 uhoh.close()
479 raise DbfError("unknown error: memo not saved")
480 return thismemo
483 "Visual Foxpro 6 specific"
484 if yo.meta.ondisk and not yo.meta.ignorememos:
485 yo.record_header_length = 8
486 if yo.meta.newmemofile:
487 if yo.meta.memo_size == 0:
488 yo.meta.memo_size = 1
489 elif 1 < yo.meta.memo_size < 33:
490 yo.meta.memo_size *= 512
491 yo.meta.mfd = open(yo.meta.memoname, 'w+b')
492 nextmemo = 512 // yo.meta.memo_size
493 if nextmemo * yo.meta.memo_size < 512:
494 nextmemo += 1
495 yo.nextmemo = nextmemo
496 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \
497 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504)
498 else:
499 try:
500 yo.meta.mfd = open(yo.meta.memoname, 'r+b')
501 yo.meta.mfd.seek(0)
502 header = yo.meta.mfd.read(512)
503 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True)
504 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True)
505 except:
506 raise DbfError("memo file appears to be corrupt")
508 yo.meta.mfd.seek(block * yo.meta.memo_size)
509 header = yo.meta.mfd.read(8)
510 length = io.unpackLongInt(header[4:], bigendian=True)
511 return yo.meta.mfd.read(length)
513 data = data.rstrip()
514 yo.meta.mfd.seek(0)
515 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True)
516 yo.meta.mfd.seek(0)
517 length = len(data) + yo.record_header_length
518 blocks = length // yo.meta.memo_size
519 if length % yo.meta.memo_size:
520 blocks += 1
521 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True))
522 yo.meta.mfd.seek(thismemo*yo.meta.memo_size)
523 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data)
524 return thismemo
525
527 """Provides a framework for dbf style tables."""
528 _version = 'basic memory table'
529 _versionabbv = 'dbf'
530 _fieldtypes = {
531 'D' : { 'Type':'Date', 'Init':io.addDate, 'Blank':Date.today, 'Retrieve':io.retrieveDate, 'Update':io.updateDate, },
532 'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool, 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, },
533 'M' : { 'Type':'Memo', 'Init':io.addMemo, 'Blank':str, 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, } }
534 _memoext = ''
535 _memotypes = tuple('M', )
536 _memoClass = _DbfMemo
537 _yesMemoMask = ''
538 _noMemoMask = ''
539 _fixed_fields = ('M','D','L')
540 _variable_fields = tuple()
541 _character_fields = tuple('M', )
542 _decimal_fields = tuple()
543 _numeric_fields = tuple()
544 _dbfTableHeader = array('c', '\x00' * 32)
545 _dbfTableHeader[0] = '\x00'
546 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
547 _dbfTableHeader[10] = '\x01'
548 _dbfTableHeader[29] = '\x00'
549 _dbfTableHeader = _dbfTableHeader.tostring()
550 _dbfTableHeaderExtra = ''
551 _supported_tables = []
552 _read_only = False
553 _meta_only = False
554 _use_deleted = True
555 backup = False
557 "implements the weakref structure for DbfLists"
561 yo._lists = set([s for s in yo._lists if s() is not None])
562 return (s() for s in yo._lists if s() is not None)
564 yo._lists = set([s for s in yo._lists if s() is not None])
565 return len(yo._lists)
566 - def add(yo, new_list):
567 yo._lists.add(weakref.ref(new_list))
568 yo._lists = set([s for s in yo._lists if s() is not None])
570 "implements the weakref structure for seperate indexes"
574 yo._indexen = set([s for s in yo._indexen if s() is not None])
575 return (s() for s in yo._indexen if s() is not None)
577 yo._indexen = set([s for s in yo._indexen if s() is not None])
578 return len(yo._indexen)
579 - def add(yo, new_list):
580 yo._indexen.add(weakref.ref(new_list))
581 yo._indexen = set([s for s in yo._indexen if s() is not None])
596 if len(data) != 32:
597 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data))
598 yo._data = array('c', data + '\x0d')
600 "get/set code page of table"
601 if cp is None:
602 return yo._data[29]
603 else:
604 cp, sd, ld = _codepage_lookup(cp)
605 yo._data[29] = cp
606 return cp
607 @property
613 @data.setter
615 if len(bytes) < 32:
616 raise DbfError("length for data of %d is less than 32" % len(bytes))
617 yo._data[:] = array('c', bytes)
618 @property
620 "extra dbf info (located after headers, before data records)"
621 fieldblock = yo._data[32:]
622 for i in range(len(fieldblock)//32+1):
623 cr = i * 32
624 if fieldblock[cr] == '\x0d':
625 break
626 else:
627 raise DbfError("corrupt field structure")
628 cr += 33
629 return yo._data[cr:].tostring()
630 @extra.setter
632 fieldblock = yo._data[32:]
633 for i in range(len(fieldblock)//32+1):
634 cr = i * 32
635 if fieldblock[cr] == '\x0d':
636 break
637 else:
638 raise DbfError("corrupt field structure")
639 cr += 33
640 yo._data[cr:] = array('c', data)
641 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
642 @property
644 "number of fields (read-only)"
645 fieldblock = yo._data[32:]
646 for i in range(len(fieldblock)//32+1):
647 cr = i * 32
648 if fieldblock[cr] == '\x0d':
649 break
650 else:
651 raise DbfError("corrupt field structure")
652 return len(fieldblock[:cr]) // 32
653 @property
655 "field block structure"
656 fieldblock = yo._data[32:]
657 for i in range(len(fieldblock)//32+1):
658 cr = i * 32
659 if fieldblock[cr] == '\x0d':
660 break
661 else:
662 raise DbfError("corrupt field structure")
663 return fieldblock[:cr].tostring()
664 @fields.setter
666 fieldblock = yo._data[32:]
667 for i in range(len(fieldblock)//32+1):
668 cr = i * 32
669 if fieldblock[cr] == '\x0d':
670 break
671 else:
672 raise DbfError("corrupt field structure")
673 cr += 32
674 fieldlen = len(block)
675 if fieldlen % 32 != 0:
676 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen)
677 yo._data[32:cr] = array('c', block)
678 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
679 fieldlen = fieldlen // 32
680 recordlen = 1
681 for i in range(fieldlen):
682 recordlen += ord(block[i*32+16])
683 yo._data[10:12] = array('c', io.packShortInt(recordlen))
684 @property
686 "number of records (maximum 16,777,215)"
687 return io.unpackLongInt(yo._data[4:8].tostring())
688 @record_count.setter
691 @property
693 "length of a record (read_only) (max of 65,535)"
694 return io.unpackShortInt(yo._data[10:12].tostring())
695 @property
697 "starting position of first record in file (must be within first 64K)"
698 return io.unpackShortInt(yo._data[8:10].tostring())
699 @start.setter
702 @property
704 "date of last table modification (read-only)"
705 return io.unpackDate(yo._data[1:4].tostring())
706 @property
708 "dbf version"
709 return yo._data[0]
710 @version.setter
714 "implements the weakref table for records"
716 yo._meta = meta
717 yo._weakref_list = [weakref.ref(lambda x: None)] * count
719 maybe = yo._weakref_list[index]()
720 if maybe is None:
721 if index < 0:
722 index += yo._meta.header.record_count
723 size = yo._meta.header.record_length
724 location = index * size + yo._meta.header.start
725 yo._meta.dfd.seek(location)
726 if yo._meta.dfd.tell() != location:
727 raise ValueError("unable to seek to offset %d in file" % location)
728 bytes = yo._meta.dfd.read(size)
729 if not bytes:
730 raise ValueError("unable to read record data from %s at location %d" % (yo._meta.filename, location))
731 maybe = _DbfRecord(recnum=index, layout=yo._meta, kamikaze=bytes, _fromdisk=True)
732 yo._weakref_list[index] = weakref.ref(maybe)
733 return maybe
735 yo._weakref_list.append(weakref.ref(record))
737 yo._weakref_list[:] = []
739 "returns records using current index"
741 yo._table = table
742 yo._index = -1
743 yo._more_records = True
747 while yo._more_records:
748 yo._index += 1
749 if yo._index >= len(yo._table):
750 yo._more_records = False
751 continue
752 record = yo._table[yo._index]
753 if not yo._table.use_deleted and record.has_been_deleted:
754 continue
755 return record
756 else:
757 raise StopIteration
759 "constructs fieldblock for disk table"
760 fieldblock = array('c', '')
761 memo = False
762 yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask))
763 for field in yo._meta.fields:
764 if yo._meta.fields.count(field) > 1:
765 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)")
766 fielddef = array('c', '\x00' * 32)
767 fielddef[:11] = array('c', io.packStr(field))
768 fielddef[11] = yo._meta[field]['type']
769 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start']))
770 fielddef[16] = chr(yo._meta[field]['length'])
771 fielddef[17] = chr(yo._meta[field]['decimals'])
772 fielddef[18] = chr(yo._meta[field]['flags'])
773 fieldblock.extend(fielddef)
774 if yo._meta[field]['type'] in yo._meta.memotypes:
775 memo = True
776 yo._meta.header.fields = fieldblock.tostring()
777 if memo:
778 yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask))
779 if yo._meta.memo is None:
780 yo._meta.memo = yo._memoClass(yo._meta)
782 "dBase III specific"
783 if yo._meta.header.version == '\x83':
784 try:
785 yo._meta.memo = yo._memoClass(yo._meta)
786 except:
787 yo._meta.dfd.close()
788 yo._meta.dfd = None
789 raise
790 if not yo._meta.ignorememos:
791 for field in yo._meta.fields:
792 if yo._meta[field]['type'] in yo._memotypes:
793 if yo._meta.header.version != '\x83':
794 yo._meta.dfd.close()
795 yo._meta.dfd = None
796 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
797 elif not os.path.exists(yo._meta.memoname):
798 yo._meta.dfd.close()
799 yo._meta.dfd = None
800 raise DbfError("Table structure corrupt: memo fields exist without memo file")
801 break
803 "builds the FieldList of names, types, and descriptions from the disk file"
804 yo._meta.fields[:] = []
805 offset = 1
806 fieldsdef = yo._meta.header.fields
807 if len(fieldsdef) % 32 != 0:
808 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
809 if len(fieldsdef) // 32 != yo.field_count:
810 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
811 for i in range(yo.field_count):
812 fieldblock = fieldsdef[i*32:(i+1)*32]
813 name = io.unpackStr(fieldblock[:11])
814 type = fieldblock[11]
815 if not type in yo._meta.fieldtypes:
816 raise DbfError("Unknown field type: %s" % type)
817 start = offset
818 length = ord(fieldblock[16])
819 offset += length
820 end = start + length
821 decimals = ord(fieldblock[17])
822 flags = ord(fieldblock[18])
823 if name in yo._meta.fields:
824 raise DbfError('Duplicate field name found: %s' % name)
825 yo._meta.fields.append(name)
826 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
828 "Returns field information Name Type(Length[,Decimals])"
829 name = yo._meta.fields[i]
830 type = yo._meta[name]['type']
831 length = yo._meta[name]['length']
832 decimals = yo._meta[name]['decimals']
833 if type in yo._decimal_fields:
834 description = "%s %s(%d,%d)" % (name, type, length, decimals)
835 elif type in yo._fixed_fields:
836 description = "%s %s" % (name, type)
837 else:
838 description = "%s %s(%d)" % (name, type, length)
839 return description
841 "loads the records from disk to memory"
842 if yo._meta_only:
843 raise DbfError("%s has been closed, records are unavailable" % yo.filename)
844 dfd = yo._meta.dfd
845 header = yo._meta.header
846 dfd.seek(header.start)
847 allrecords = dfd.read()
848 dfd.seek(0)
849 length = header.record_length
850 for i in range(header.record_count):
851 record_data = allrecords[length*i:length*i+length]
852 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True))
853 dfd.seek(0)
855 if specs is None:
856 specs = yo.field_names
857 elif isinstance(specs, str):
858 specs = specs.split(sep)
859 else:
860 specs = list(specs)
861 specs = [s.strip() for s in specs]
862 return specs
864 "synchronizes the disk file with current data"
865 if yo._meta.inmemory:
866 return
867 fd = yo._meta.dfd
868 fd.seek(0)
869 fd.write(yo._meta.header.data)
870 if not headeronly:
871 for record in yo._table:
872 record._update_disk()
873 fd.flush()
874 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length)
875 if 'db3' in yo._versionabbv:
876 fd.seek(0, os.SEEK_END)
877 fd.write('\x1a')
878 fd.flush()
879 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length + 1)
880
888 if name in ('_table'):
889 if yo._meta.ondisk:
890 yo._table = yo._Table(len(yo), yo._meta)
891 else:
892 yo._table = []
893 yo._loadtable()
894 return object.__getattribute__(yo, name)
896 if type(value) == int:
897 if not -yo._meta.header.record_count <= value < yo._meta.header.record_count:
898 raise IndexError("Record %d is not in table." % value)
899 return yo._table[value]
900 elif type(value) == slice:
901 sequence = List(desc='%s --> %s' % (yo.filename, value), field_names=yo.field_names)
902 yo._dbflists.add(sequence)
903 for index in range(len(yo))[value]:
904 record = yo._table[index]
905 if yo.use_deleted is True or not record.has_been_deleted:
906 sequence.append(record)
907 return sequence
908 else:
909 raise TypeError('type <%s> not valid for indexing' % type(value))
910 - def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False,
911 read_only=False, keep_memos=False, meta_only=False, codepage=None):
912 """open/create dbf file
913 filename should include path if needed
914 field_specs can be either a ;-delimited string or a list of strings
915 memo_size is always 512 for db3 memos
916 ignore_memos is useful if the memo file is missing or corrupt
917 read_only will load records into memory, then close the disk file
918 keep_memos will also load any memo fields into memory
919 meta_only will ignore all records, keeping only basic table information
920 codepage will override whatever is set in the table itself"""
921 if filename[0] == filename[-1] == ':':
922 if field_specs is None:
923 raise DbfError("field list must be specified for memory tables")
924 elif type(yo) is DbfTable:
925 raise DbfError("only memory tables supported")
926 yo._dbflists = yo._DbfLists()
927 yo._indexen = yo._Indexen()
928 yo._meta = meta = yo._MetaData()
929 meta.table = weakref.ref(yo)
930 meta.filename = filename
931 meta.fields = []
932 meta.fieldtypes = yo._fieldtypes
933 meta.fixed_fields = yo._fixed_fields
934 meta.variable_fields = yo._variable_fields
935 meta.character_fields = yo._character_fields
936 meta.decimal_fields = yo._decimal_fields
937 meta.numeric_fields = yo._numeric_fields
938 meta.memotypes = yo._memotypes
939 meta.ignorememos = ignore_memos
940 meta.memo_size = memo_size
941 meta.input_decoder = codecs.getdecoder(input_decoding)
942 meta.output_encoder = codecs.getencoder(input_decoding)
943 meta.return_ascii = return_ascii
944 meta.header = header = yo._TableHeader(yo._dbfTableHeader)
945 header.extra = yo._dbfTableHeaderExtra
946 header.data
947 if filename[0] == filename[-1] == ':':
948 yo._table = []
949 meta.ondisk = False
950 meta.inmemory = True
951 meta.memoname = filename
952 else:
953 base, ext = os.path.splitext(filename)
954 if ext == '':
955 meta.filename = base + '.dbf'
956 meta.memoname = base + yo._memoext
957 meta.ondisk = True
958 meta.inmemory = False
959 if field_specs:
960 if meta.ondisk:
961 meta.dfd = open(meta.filename, 'w+b')
962 meta.newmemofile = True
963 yo.add_fields(field_specs)
964 header.codepage(codepage or default_codepage)
965 cp, sd, ld = _codepage_lookup(meta.header.codepage())
966 meta.decoder = codecs.getdecoder(sd)
967 meta.encoder = codecs.getencoder(sd)
968 return
969 try:
970 dfd = meta.dfd = open(meta.filename, 'r+b')
971 except IOError, e:
972 raise DbfError(str(e))
973 dfd.seek(0)
974 meta.header = header = yo._TableHeader(dfd.read(32))
975 if not header.version in yo._supported_tables:
976 dfd.close()
977 dfd = None
978 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version)))
979 cp, sd, ld = _codepage_lookup(meta.header.codepage())
980 yo._meta.decoder = codecs.getdecoder(sd)
981 yo._meta.encoder = codecs.getencoder(sd)
982 fieldblock = dfd.read(header.start - 32)
983 for i in range(len(fieldblock)//32+1):
984 fieldend = i * 32
985 if fieldblock[fieldend] == '\x0d':
986 break
987 else:
988 raise DbfError("corrupt field structure in header")
989 if len(fieldblock[:fieldend]) % 32 != 0:
990 raise DbfError("corrupt field structure in header")
991 header.fields = fieldblock[:fieldend]
992 header.extra = fieldblock[fieldend+1:]
993 yo._initializeFields()
994 yo._checkMemoIntegrity()
995 meta.current = -1
996 if len(yo) > 0:
997 meta.current = 0
998 dfd.seek(0)
999 if meta_only:
1000 yo.close(keep_table=False, keep_memos=False)
1001 elif read_only:
1002 yo.close(keep_table=True, keep_memos=keep_memos)
1003 if codepage is not None:
1004 cp, sd, ld = _codepage_lookup(codepage)
1005 yo._meta.decoder = codecs.getdecoder(sd)
1006 yo._meta.encoder = codecs.getencoder(sd)
1007
1015 if yo._read_only:
1016 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename
1017 elif yo._meta_only:
1018 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename
1019 else:
1020 return __name__ + ".Table('%s')" % yo._meta.filename
1022 if yo._read_only:
1023 status = "read-only"
1024 elif yo._meta_only:
1025 status = "meta-only"
1026 else:
1027 status = "read/write"
1028 str = """
1029 Table: %s
1030 Type: %s
1031 Codepage: %s
1032 Status: %s
1033 Last updated: %s
1034 Record count: %d
1035 Field count: %d
1036 Record length: %d """ % (yo.filename, version_map.get(yo._meta.header.version,
1037 'unknown - ' + hex(ord(yo._meta.header.version))), yo.codepage, status,
1038 yo.last_update, len(yo), yo.field_count, yo.record_length)
1039 str += "\n --Fields--\n"
1040 for i in range(len(yo._meta.fields)):
1041 str += "%11d) %s\n" % (i, yo._fieldLayout(i))
1042 return str
1043 @property
1045 return "%s (%s)" % code_pages[yo._meta.header.codepage()]
1046 @codepage.setter
1047 - def codepage(yo, cp):
1048 cp = code_pages[yo._meta.header.codepage(cp)][0]
1049 yo._meta.decoder = codecs.getdecoder(cp)
1050 yo._meta.encoder = codecs.getencoder(cp)
1051 yo._update_disk(headeronly=True)
1052 @property
1054 "the number of fields in the table"
1055 return yo._meta.header.field_count
1056 @property
1058 "a list of the fields in the table"
1059 return yo._meta.fields[:]
1060 @property
1062 "table's file name, including path (if specified on open)"
1063 return yo._meta.filename
1064 @property
1066 "date of last update"
1067 return yo._meta.header.update
1068 @property
1070 "table's memo name (if path included in filename on open)"
1071 return yo._meta.memoname
1072 @property
1074 "number of bytes in a record"
1075 return yo._meta.header.record_length
1076 @property
1078 "index number of the current record"
1079 return yo._meta.current
1080 @property
1084 @property
1086 "process or ignore deleted records"
1087 return yo._use_deleted
1088 @use_deleted.setter
1091 @property
1093 "returns the dbf type of the table"
1094 return yo._version
1096 """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]]
1097 backup table is created with _backup appended to name
1098 then modifies current structure"""
1099 all_records = [record for record in yo]
1100 if yo:
1101 yo.create_backup()
1102 yo._meta.blankrecord = None
1103 meta = yo._meta
1104 offset = meta.header.record_length
1105 fields = yo._list_fields(field_specs, sep=';')
1106 for field in fields:
1107 try:
1108 name, format = field.split()
1109 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum():
1110 raise DbfError("%s invalid: field names must start with a letter, and can only contain letters, digits, and _" % name)
1111 name = name.lower()
1112 if name in meta.fields:
1113 raise DbfError("Field '%s' already exists" % name)
1114 field_type = format[0].upper()
1115 if len(name) > 10:
1116 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name)))
1117 if not field_type in meta.fieldtypes.keys():
1118 raise DbfError("Unknown field type: %s" % field_type)
1119 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format)
1120 except ValueError:
1121 raise DbfError("invalid field specifier: %s" % field)
1122 start = offset
1123 end = offset + length
1124 offset = end
1125 meta.fields.append(name)
1126 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0}
1127 if meta[name]['type'] in yo._memotypes and meta.memo is None:
1128 meta.memo = yo._memoClass(meta)
1129 for record in yo:
1130 record[name] = meta.fieldtypes[field_type]['Blank']()
1131 yo._buildHeaderFields()
1132 yo._update_disk()
1133 - def append(yo, kamikaze='', drop=False, multiple=1):
1134 "adds <multiple> blank records, and fills fields with dict/tuple values if present"
1135 if not yo.field_count:
1136 raise DbfError("No fields defined, cannot append")
1137 empty_table = len(yo) == 0
1138 dictdata = False
1139 tupledata = False
1140 if not isinstance(kamikaze, _DbfRecord):
1141 if isinstance(kamikaze, dict):
1142 dictdata = kamikaze
1143 kamikaze = ''
1144 elif isinstance(kamikaze, tuple):
1145 tupledata = kamikaze
1146 kamikaze = ''
1147 newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze)
1148 yo._table.append(newrecord)
1149 yo._meta.header.record_count += 1
1150 if dictdata:
1151 newrecord.gather_fields(dictdata, drop=drop)
1152 elif tupledata:
1153 for index, item in enumerate(tupledata):
1154 newrecord[index] = item
1155 elif kamikaze == str:
1156 for field in yo._meta.memofields:
1157 newrecord[field] = ''
1158 elif kamikaze:
1159 for field in yo._meta.memofields:
1160 newrecord[field] = kamikaze[field]
1161 newrecord.write_record()
1162 multiple -= 1
1163 if multiple:
1164 data = newrecord._data
1165 single = yo._meta.header.record_count
1166 total = single + multiple
1167 while single < total:
1168 multi_record = _DbfRecord(single, yo._meta, kamikaze=data)
1169 yo._table.append(multi_record)
1170 for field in yo._meta.memofields:
1171 multi_record[field] = newrecord[field]
1172 single += 1
1173 multi_record.write_record()
1174 yo._meta.header.record_count = total
1175 yo._meta.current = yo._meta.header.record_count - 1
1176 newrecord = multi_record
1177 yo._update_disk(headeronly=True)
1178 if empty_table:
1179 yo._meta.current = 0
1180 return newrecord
1181 - def bof(yo, _move=False):
1196 - def bottom(yo, get_record=False):
1197 """sets record pointer to bottom of table
1198 if get_record, seeks to and returns last (non-deleted) record
1199 DbfError if table is empty
1200 Bof if all records deleted and use_deleted is False"""
1201 yo._meta.current = yo._meta.header.record_count
1202 if get_record:
1203 try:
1204 return yo.prev()
1205 except Bof:
1206 yo._meta.current = yo._meta.header.record_count
1207 raise Eof()
1208 - def close(yo, keep_table=False, keep_memos=False):
1209 """closes disk files
1210 ensures table data is available if keep_table
1211 ensures memo data is available if keep_memos"""
1212 yo._meta.inmemory = True
1213 if keep_table:
1214 replacement_table = []
1215 for record in yo._table:
1216 replacement_table.append(record)
1217 yo._table = replacement_table
1218 else:
1219 if yo._meta.ondisk:
1220 yo._meta_only = True
1221 if yo._meta.mfd is not None:
1222 if not keep_memos:
1223 yo._meta.ignorememos = True
1224 else:
1225 memo_fields = []
1226 for field in yo.field_names:
1227 if yo.is_memotype(field):
1228 memo_fields.append(field)
1229 for record in yo:
1230 for field in memo_fields:
1231 record[field] = record[field]
1232 yo._meta.mfd.close()
1233 yo._meta.mfd = None
1234 if yo._meta.ondisk:
1235 yo._meta.dfd.close()
1236 yo._meta.dfd = None
1237 if keep_table:
1238 yo._read_only = True
1239 yo._meta.ondisk = False
1241 "creates a backup table -- ignored if memory table"
1242 if yo.filename[0] == yo.filename[-1] == ':':
1243 return
1244 if new_name is None:
1245 new_name = os.path.splitext(yo.filename)[0] + '_backup.dbf'
1246 else:
1247 overwrite = True
1248 if overwrite or not yo.backup:
1249 bkup = open(new_name, 'wb')
1250 try:
1251 yo._meta.dfd.seek(0)
1252 copyfileobj(yo._meta.dfd, bkup)
1253 yo.backup = new_name
1254 finally:
1255 bkup.close()
1259 "returns current logical record, or its index"
1260 if yo._meta.current < 0:
1261 raise Bof()
1262 elif yo._meta.current >= yo._meta.header.record_count:
1263 raise Eof()
1264 if index:
1265 return yo._meta.current
1266 return yo._table[yo._meta.current]
1268 """removes field(s) from the table
1269 creates backup files with _backup appended to the file name,
1270 then modifies current structure"""
1271 doomed = yo._list_fields(doomed)
1272 for victim in doomed:
1273 if victim not in yo._meta.fields:
1274 raise DbfError("field %s not in table -- delete aborted" % victim)
1275 all_records = [record for record in yo]
1276 yo.create_backup()
1277 for victim in doomed:
1278 yo._meta.fields.pop(yo._meta.fields.index(victim))
1279 start = yo._meta[victim]['start']
1280 end = yo._meta[victim]['end']
1281 for record in yo:
1282 record._data = record._data[:start] + record._data[end:]
1283 for field in yo._meta.fields:
1284 if yo._meta[field]['start'] == end:
1285 end = yo._meta[field]['end']
1286 yo._meta[field]['start'] = start
1287 yo._meta[field]['end'] = start + yo._meta[field]['length']
1288 start = yo._meta[field]['end']
1289 yo._buildHeaderFields()
1290 yo._update_disk()
1291 - def eof(yo, _move=False):
1306 - def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
1307 """writes the table using CSV or tab-delimited format, using the filename
1308 given if specified, otherwise the table name"""
1309 if filename is not None:
1310 path, filename = os.path.split(filename)
1311 else:
1312 path, filename = os.path.split(yo.filename)
1313 filename = os.path.join(path, filename)
1314 field_specs = yo._list_fields(field_specs)
1315 if records is None:
1316 records = yo
1317 format = format.lower()
1318 if format not in ('csv', 'tab', 'fixed'):
1319 raise DbfError("export format: csv, tab, or fixed -- not %s" % format)
1320 if format == 'fixed':
1321 format = 'txt'
1322 base, ext = os.path.splitext(filename)
1323 if ext.lower() in ('', '.dbf'):
1324 filename = base + "." + format[:3]
1325 fd = open(filename, 'w')
1326 try:
1327 if format == 'csv':
1328 csvfile = csv.writer(fd, dialect='dbf')
1329 if header:
1330 csvfile.writerow(field_specs)
1331 for record in records:
1332 fields = []
1333 for fieldname in field_specs:
1334 fields.append(record[fieldname])
1335 csvfile.writerow(fields)
1336 elif format == 'tab':
1337 if header:
1338 fd.write('\t'.join(field_specs) + '\n')
1339 for record in records:
1340 fields = []
1341 for fieldname in field_specs:
1342 fields.append(str(record[fieldname]))
1343 fd.write('\t'.join(fields) + '\n')
1344 else:
1345 header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w')
1346 header.write("%-15s Size\n" % "Field Name")
1347 header.write("%-15s ----\n" % ("-" * 15))
1348 sizes = []
1349 for field in field_specs:
1350 size = yo.size(field)[0]
1351 sizes.append(size)
1352 header.write("%-15s %3d\n" % (field, size))
1353 header.write('\nTotal Records in file: %d\n' % len(records))
1354 header.close()
1355 for record in records:
1356 fields = []
1357 for i, field_name in enumerate(field_specs):
1358 fields.append("%-*s" % (sizes[i], record[field_name]))
1359 fd.write(''.join(fields) + '\n')
1360 finally:
1361 fd.close()
1362 fd = None
1363 return len(records)
1365 "returns record at physical_index[recno]"
1366 return yo._table[recno]
1367 - def goto(yo, criteria):
1368 """changes the record pointer to the first matching (non-deleted) record
1369 criteria should be either a tuple of tuple(value, field, func) triples,
1370 or an integer to go to"""
1371 if isinstance(criteria, int):
1372 if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count:
1373 raise IndexError("Record %d does not exist" % criteria)
1374 if criteria < 0:
1375 criteria += yo._meta.header.record_count
1376 yo._meta.current = criteria
1377 return yo.current()
1378 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop])
1379 specs = tuple([(field, func) for value, field, func in criteria])
1380 match = tuple([value for value, field, func in criteria])
1381 current = yo.current(index=True)
1382 matchlen = len(match)
1383 while not yo.Eof():
1384 record = yo.current()
1385 results = record(*specs)
1386 if results == match:
1387 return record
1388 return yo.goto(current)
1390 "returns True if name is a variable-length field type"
1391 return yo._meta[name]['type'] in yo._decimal_fields
1393 "returns True if name is a memo type field"
1394 return yo._meta[name]['type'] in yo._memotypes
1395 - def new(yo, filename, field_specs=None, codepage=None):
1409 "set record pointer to next (non-deleted) record, and return it"
1410 if yo.eof(_move=True):
1411 raise Eof()
1412 return yo.current()
1448
1449 - def pack(yo, _pack=True):
1450 "physically removes all deleted records"
1451 for dbfindex in yo._indexen:
1452 dbfindex.clear()
1453 newtable = []
1454 index = 0
1455 offset = 0
1456 for record in yo._table:
1457 found = False
1458 if record.has_been_deleted and _pack:
1459 for dbflist in yo._dbflists:
1460 if dbflist._purge(record, record.record_number - offset, 1):
1461 found = True
1462 record._recnum = -1
1463 else:
1464 record._recnum = index
1465 newtable.append(record)
1466 index += 1
1467 if found:
1468 offset += 1
1469 found = False
1470 yo._table.clear()
1471 for record in newtable:
1472 yo._table.append(record)
1473 yo._meta.header.record_count = index
1474 yo._current = -1
1475 yo._update_disk()
1476 yo.reindex()
1478 "set record pointer to previous (non-deleted) record, and return it"
1479 if yo.bof(_move=True):
1480 raise Bof
1481 return yo.current()
1482 - def query(yo, sql_command=None, python=None):
1483 "uses exec to perform queries on the table"
1484 if sql_command:
1485 return sql(yo, sql_command)
1486 elif python is None:
1487 raise DbfError("query: python parameter must be specified")
1488 possible = List(desc="%s --> %s" % (yo.filename, python), field_names=yo.field_names)
1489 yo._dbflists.add(possible)
1490 query_result = {}
1491 select = 'query_result["keep"] = %s' % python
1492 g = {}
1493 use_deleted = yo.use_deleted
1494 for record in yo:
1495 query_result['keep'] = False
1496 g['query_result'] = query_result
1497 exec select in g, record
1498 if query_result['keep']:
1499 possible.append(record)
1500 record.write_record()
1501 return possible
1503 for dbfindex in yo._indexen:
1504 dbfindex.reindex()
1506 "renames an existing field"
1507 if yo:
1508 yo.create_backup()
1509 if not oldname in yo._meta.fields:
1510 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname)
1511 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum():
1512 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits")
1513 newname = newname.lower()
1514 if newname in yo._meta.fields:
1515 raise DbfError("field --%s-- already exists" % newname)
1516 if len(newname) > 10:
1517 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname)))
1518 yo._meta[newname] = yo._meta[oldname]
1519 yo._meta.fields[yo._meta.fields.index(oldname)] = newname
1520 yo._buildHeaderFields()
1521 yo._update_disk(headeronly=True)
1522 - def size(yo, field):
1523 "returns size of field as a tuple of (length, decimals)"
1524 if field in yo:
1525 return (yo._meta[field]['length'], yo._meta[field]['decimals'])
1526 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1528 """return list of fields suitable for creating same table layout
1529 @param fields: list of fields or None for all fields"""
1530 field_specs = []
1531 fields = yo._list_fields(fields)
1532 try:
1533 for name in fields:
1534 field_specs.append(yo._fieldLayout(yo.field_names.index(name)))
1535 except ValueError:
1536 raise DbfError("field --%s-- does not exist" % name)
1537 return field_specs
1538 - def top(yo, get_record=False):
1539 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record
1540 DbfError if table is empty
1541 Eof if all records are deleted and use_deleted is False"""
1542 yo._meta.current = -1
1543 if get_record:
1544 try:
1545 return yo.next()
1546 except Eof:
1547 yo._meta.current = -1
1548 raise Bof()
1549 - def type(yo, field):
1550 "returns type of field"
1551 if field in yo:
1552 return yo._meta[field]['type']
1553 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1554 - def zap(yo, areyousure=False):
1555 """removes all records from table -- this cannot be undone!
1556 areyousure must be True, else error is raised"""
1557 if areyousure:
1558 if yo._meta.inmemory:
1559 yo._table = []
1560 else:
1561 yo._table.clear()
1562 yo._meta.header.record_count = 0
1563 yo._current = -1
1564 yo._update_disk()
1565 else:
1566 raise DbfError("You must say you are sure to wipe the table")
1568 """Provides an interface for working with dBase III tables."""
1569 _version = 'dBase III Plus'
1570 _versionabbv = 'db3'
1571 _fieldtypes = {
1572 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
1573 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
1574 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
1575 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1576 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric} }
1577 _memoext = '.dbt'
1578 _memotypes = ('M',)
1579 _memoClass = _Db3Memo
1580 _yesMemoMask = '\x80'
1581 _noMemoMask = '\x7f'
1582 _fixed_fields = ('D','L','M')
1583 _variable_fields = ('C','N')
1584 _character_fields = ('C','M')
1585 _decimal_fields = ('N',)
1586 _numeric_fields = ('N',)
1587 _dbfTableHeader = array('c', '\x00' * 32)
1588 _dbfTableHeader[0] = '\x03'
1589 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
1590 _dbfTableHeader[10] = '\x01'
1591 _dbfTableHeader[29] = '\x03'
1592 _dbfTableHeader = _dbfTableHeader.tostring()
1593 _dbfTableHeaderExtra = ''
1594 _supported_tables = ['\x03', '\x83']
1595 _read_only = False
1596 _meta_only = False
1597 _use_deleted = True
1599 "dBase III specific"
1600 if yo._meta.header.version == '\x83':
1601 try:
1602 yo._meta.memo = yo._memoClass(yo._meta)
1603 except:
1604 yo._meta.dfd.close()
1605 yo._meta.dfd = None
1606 raise
1607 if not yo._meta.ignorememos:
1608 for field in yo._meta.fields:
1609 if yo._meta[field]['type'] in yo._memotypes:
1610 if yo._meta.header.version != '\x83':
1611 yo._meta.dfd.close()
1612 yo._meta.dfd = None
1613 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
1614 elif not os.path.exists(yo._meta.memoname):
1615 yo._meta.dfd.close()
1616 yo._meta.dfd = None
1617 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1618 break
1620 "builds the FieldList of names, types, and descriptions"
1621 yo._meta.fields[:] = []
1622 offset = 1
1623 fieldsdef = yo._meta.header.fields
1624 if len(fieldsdef) % 32 != 0:
1625 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1626 if len(fieldsdef) // 32 != yo.field_count:
1627 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1628 for i in range(yo.field_count):
1629 fieldblock = fieldsdef[i*32:(i+1)*32]
1630 name = io.unpackStr(fieldblock[:11])
1631 type = fieldblock[11]
1632 if not type in yo._meta.fieldtypes:
1633 raise DbfError("Unknown field type: %s" % type)
1634 start = offset
1635 length = ord(fieldblock[16])
1636 offset += length
1637 end = start + length
1638 decimals = ord(fieldblock[17])
1639 flags = ord(fieldblock[18])
1640 yo._meta.fields.append(name)
1641 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1643 'Provides an interface for working with FoxPro 2 tables'
1644 _version = 'Foxpro'
1645 _versionabbv = 'fp'
1646 _fieldtypes = {
1647 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
1648 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
1649 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
1650 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
1651 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
1652 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo},
1653 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1654 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1655 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
1656 _memoext = '.fpt'
1657 _memotypes = ('G','M','P')
1658 _memoClass = _VfpMemo
1659 _yesMemoMask = '\xf5'
1660 _noMemoMask = '\x03'
1661 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1662 _variable_fields = ('C','F','N')
1663 _character_fields = ('C','M')
1664 _decimal_fields = ('F','N')
1665 _numeric_fields = ('B','F','I','N','Y')
1666 _supported_tables = ('\x03', '\xf5')
1667 _dbfTableHeader = array('c', '\x00' * 32)
1668 _dbfTableHeader[0] = '\x30'
1669 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1670 _dbfTableHeader[10] = '\x01'
1671 _dbfTableHeader[29] = '\x03'
1672 _dbfTableHeader = _dbfTableHeader.tostring()
1673 _dbfTableHeaderExtra = '\x00' * 263
1674 _use_deleted = True
1676 if os.path.exists(yo._meta.memoname):
1677 try:
1678 yo._meta.memo = yo._memoClass(yo._meta)
1679 except:
1680 yo._meta.dfd.close()
1681 yo._meta.dfd = None
1682 raise
1683 if not yo._meta.ignorememos:
1684 for field in yo._meta.fields:
1685 if yo._meta[field]['type'] in yo._memotypes:
1686 if not os.path.exists(yo._meta.memoname):
1687 yo._meta.dfd.close()
1688 yo._meta.dfd = None
1689 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1690 break
1692 "builds the FieldList of names, types, and descriptions"
1693 yo._meta.fields[:] = []
1694 offset = 1
1695 fieldsdef = yo._meta.header.fields
1696 if len(fieldsdef) % 32 != 0:
1697 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1698 if len(fieldsdef) // 32 != yo.field_count:
1699 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1700 for i in range(yo.field_count):
1701 fieldblock = fieldsdef[i*32:(i+1)*32]
1702 name = io.unpackStr(fieldblock[:11])
1703 type = fieldblock[11]
1704 if not type in yo._meta.fieldtypes:
1705 raise DbfError("Unknown field type: %s" % type)
1706 elif type == '0':
1707 return
1708 start = offset
1709 length = ord(fieldblock[16])
1710 offset += length
1711 end = start + length
1712 decimals = ord(fieldblock[17])
1713 flags = ord(fieldblock[18])
1714 yo._meta.fields.append(name)
1715 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1716
1718 'Provides an interface for working with Visual FoxPro 6 tables'
1719 _version = 'Visual Foxpro v6'
1720 _versionabbv = 'vfp'
1721 _fieldtypes = {
1722 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
1723 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
1724 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
1725 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
1726 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
1727 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
1728 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
1729 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
1730 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
1731 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
1732 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
1733 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
1734 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
1735 _memoext = '.fpt'
1736 _memotypes = ('G','M','P')
1737 _memoClass = _VfpMemo
1738 _yesMemoMask = '\x30'
1739 _noMemoMask = '\x30'
1740 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1741 _variable_fields = ('C','F','N')
1742 _character_fields = ('C','M')
1743 _decimal_fields = ('F','N')
1744 _numeric_fields = ('B','F','I','N','Y')
1745 _supported_tables = ('\x30',)
1746 _dbfTableHeader = array('c', '\x00' * 32)
1747 _dbfTableHeader[0] = '\x30'
1748 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1749 _dbfTableHeader[10] = '\x01'
1750 _dbfTableHeader[29] = '\x03'
1751 _dbfTableHeader = _dbfTableHeader.tostring()
1752 _dbfTableHeaderExtra = '\x00' * 263
1753 _use_deleted = True
1755 if os.path.exists(yo._meta.memoname):
1756 try:
1757 yo._meta.memo = yo._memoClass(yo._meta)
1758 except:
1759 yo._meta.dfd.close()
1760 yo._meta.dfd = None
1761 raise
1762 if not yo._meta.ignorememos:
1763 for field in yo._meta.fields:
1764 if yo._meta[field]['type'] in yo._memotypes:
1765 if not os.path.exists(yo._meta.memoname):
1766 yo._meta.dfd.close()
1767 yo._meta.dfd = None
1768 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1769 break
1771 "builds the FieldList of names, types, and descriptions"
1772 yo._meta.fields[:] = []
1773 offset = 1
1774 fieldsdef = yo._meta.header.fields
1775 for i in range(yo.field_count):
1776 fieldblock = fieldsdef[i*32:(i+1)*32]
1777 name = io.unpackStr(fieldblock[:11])
1778 type = fieldblock[11]
1779 if not type in yo._meta.fieldtypes:
1780 raise DbfError("Unknown field type: %s" % type)
1781 elif type == '0':
1782 return
1783 start = io.unpackLongInt(fieldblock[12:16])
1784 length = ord(fieldblock[16])
1785 offset += length
1786 end = start + length
1787 decimals = ord(fieldblock[17])
1788 flags = ord(fieldblock[18])
1789 yo._meta.fields.append(name)
1790 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1791 -class List(object):
1792 "list of Dbf records, with set-like behavior"
1793 _desc = ''
1794 - def __init__(yo, new_records=None, desc=None, key=None, field_names=None):
1795 yo.field_names = field_names
1796 yo._list = []
1797 yo._set = set()
1798 if key is not None:
1799 yo.key = key
1800 if key.__doc__ is None:
1801 key.__doc__ = 'unknown'
1802 key = yo.key
1803 yo._current = -1
1804 if isinstance(new_records, yo.__class__) and key is new_records.key:
1805 yo._list = new_records._list[:]
1806 yo._set = new_records._set.copy()
1807 yo._current = 0
1808 elif new_records is not None:
1809 for record in new_records:
1810 value = key(record)
1811 item = (record.record_table, record.record_number, value)
1812 if value not in yo._set:
1813 yo._set.add(value)
1814 yo._list.append(item)
1815 yo._current = 0
1816 if desc is not None:
1817 yo._desc = desc
1819 key = yo.key
1820 if isinstance(other, (DbfTable, list)):
1821 other = yo.__class__(other, key=key)
1822 if isinstance(other, yo.__class__):
1823 result = yo.__class__()
1824 result._set = yo._set.copy()
1825 result._list[:] = yo._list[:]
1826 result.key = yo.key
1827 if key is other.key:
1828 for item in other._list:
1829 if item[2] not in result._set:
1830 result._set.add(item[2])
1831 result._list.append(item)
1832 else:
1833 for rec in other:
1834 value = key(rec)
1835 if value not in result._set:
1836 result._set.add(value)
1837 result._list.append((rec.record_table, rec.record_number, value))
1838 result._current = 0 if result else -1
1839 return result
1840 return NotImplemented
1842 if isinstance(record, tuple):
1843 item = record
1844 else:
1845 item = yo.key(record)
1846 return item in yo._set
1848 if isinstance(key, int):
1849 item = yo._list.pop[key]
1850 yo._set.remove(item[2])
1851 elif isinstance(key, slice):
1852 yo._set.difference_update([item[2] for item in yo._list[key]])
1853 yo._list.__delitem__(key)
1854 else:
1855 raise TypeError
1857 if isinstance(key, int):
1858 count = len(yo._list)
1859 if not -count <= key < count:
1860 raise IndexError("Record %d is not in list." % key)
1861 return yo._get_record(*yo._list[key])
1862 elif isinstance(key, slice):
1863 result = yo.__class__()
1864 result._list[:] = yo._list[key]
1865 result._set = set(result._list)
1866 result.key = yo.key
1867 result._current = 0 if result else -1
1868 return result
1869 else:
1870 raise TypeError('indices must be integers')
1872 return (table.get_record(recno) for table, recno, value in yo._list)
1874 return len(yo._list)
1880 if yo._desc:
1881 return "%s(key=%s - %s - %d records)" % (yo.__class__, yo.key.__doc__, yo._desc, len(yo._list))
1882 else:
1883 return "%s(key=%s - %d records)" % (yo.__class__, yo.key.__doc__, len(yo._list))
1885 key = yo.key
1886 if isinstance(other, (DbfTable, list)):
1887 other = yo.__class__(other, key=key)
1888 if isinstance(other, yo.__class__):
1889 result = yo.__class__()
1890 result._list[:] = other._list[:]
1891 result._set = other._set.copy()
1892 result.key = key
1893 lost = set()
1894 if key is other.key:
1895 for item in yo._list:
1896 if item[2] in result._list:
1897 result._set.remove(item[2])
1898 lost.add(item)
1899 else:
1900 for rec in other:
1901 value = key(rec)
1902 if value in result._set:
1903 result._set.remove(value)
1904 lost.add((rec.record_table, rec.record_number, value))
1905 result._list = [item for item in result._list if item not in lost]
1906 result._current = 0 if result else -1
1907 return result
1908 return NotImplemented
1910 key = yo.key
1911 if isinstance(other, (DbfTable, list)):
1912 other = yo.__class__(other, key=key)
1913 if isinstance(other, yo.__class__):
1914 result = yo.__class__()
1915 result._list[:] = yo._list[:]
1916 result._set = yo._set.copy()
1917 result.key = key
1918 lost = set()
1919 if key is other.key:
1920 for item in other._list:
1921 if item[2] in result._set:
1922 result._set.remove(item[2])
1923 lost.add(item[2])
1924 else:
1925 for rec in other:
1926 value = key(rec)
1927 if value in result._set:
1928 result._set.remove(value)
1929 lost.add(value)
1930 result._list = [item for item in result._list if item[2] not in lost]
1931 result._current = 0 if result else -1
1932 return result
1933 return NotImplemented
1935 if item[2] not in yo._set:
1936 yo._set.add(item[2])
1937 yo._list.append(item)
1938 - def _get_record(yo, table=None, rec_no=None, value=None):
1939 if table is rec_no is None:
1940 table, rec_no, value = yo._list[yo._current]
1941 return table.get_record(rec_no)
1942 - def _purge(yo, record, old_record_number, offset):
1943 partial = record.record_table, old_record_number
1944 records = sorted(yo._list, key=lambda item: (item[0], item[1]))
1945 for item in records:
1946 if partial == item[:2]:
1947 found = True
1948 break
1949 elif partial[0] is item[0] and partial[1] < item[1]:
1950 found = False
1951 break
1952 else:
1953 found = False
1954 if found:
1955 yo._list.pop(yo._list.index(item))
1956 yo._set.remove(item[2])
1957 start = records.index(item) + found
1958 for item in records[start:]:
1959 if item[0] is not partial[0]:
1960 break
1961 i = yo._list.index(item)
1962 yo._set.remove(item[2])
1963 item = item[0], (item[1] - offset), item[2]
1964 yo._list[i] = item
1965 yo._set.add(item[2])
1966 return found
1973 if yo._list:
1974 yo._current = len(yo._list) - 1
1975 return yo._get_record()
1976 raise DbfError("dbf.List is empty")
1978 yo._list = []
1979 yo._set = set()
1980 yo._current = -1
1982 if yo._current < 0:
1983 raise Bof()
1984 elif yo._current == len(yo._list):
1985 raise Eof()
1986 return yo._get_record()
1987 - def extend(yo, new_records):
2003 - def goto(yo, index_number):
2004 if yo._list:
2005 if 0 <= index_number <= len(yo._list):
2006 yo._current = index_number
2007 return yo._get_record()
2008 raise DbfError("index %d not in dbf.List of %d records" % (index_number, len(yo._list)))
2009 raise DbfError("dbf.List is empty")
2010 - def index(yo, sort=None, reverse=False):
2011 "sort= ((field_name, func), (field_name, func),) | 'ORIGINAL'"
2012 if sort is None:
2013 results = []
2014 for field, func in yo._meta.index:
2015 results.append("%s(%s)" % (func.__name__, field))
2016 return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed])
2017 yo._meta.index_reversed = reverse
2018 if sort == 'ORIGINAL':
2019 yo._index = range(yo._meta.header.record_count)
2020 yo._meta.index = 'ORIGINAL'
2021 if reverse:
2022 yo._index.reverse()
2023 return
2024 new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop])
2025 yo._meta.index = tuple(new_sort)
2026 yo._meta.orderresults = [''] * len(yo)
2027 for record in yo:
2028 yo._meta.orderresults[record.record_number] = record()
2029 yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
2030 - def index(yo, record, start=None, stop=None):
2042 - def key(yo, record):
2046 if yo._current < len(yo._list):
2047 yo._current += 1
2048 if yo._current < len(yo._list):
2049 return yo._get_record()
2050 raise Eof()
2051 - def pop(yo, index=None):
2052 if index is None:
2053 table, recno, value = yo._list.pop()
2054 else:
2055 table, recno, value = yo._list.pop(index)
2056 yo._set.remove(value)
2057 return yo._get_record(table, recno, value)
2059 if yo._current >= 0:
2060 yo._current -= 1
2061 if yo._current > -1:
2062 return yo._get_record()
2063 raise Bof()
2071 if yo._list:
2072 yo._current = 0
2073 return yo._get_record()
2074 raise DbfError("dbf.List is empty")
2075 - def sort(yo, key=None, reverse=False):
2079
2091 "returns records using this index"
2093 yo.table = table
2094 yo.records = records
2095 yo.index = 0
2107 - def __init__(yo, table, key, field_names=None):
2108 yo._table = table
2109 yo._values = []
2110 yo._rec_by_val = []
2111 yo._records = {}
2112 yo.__doc__ = key.__doc__ or 'unknown'
2113 yo.key = key
2114 yo.field_names = field_names or table.field_names
2115 for record in table:
2116 value = key(record)
2117 if value is DoNotIndex:
2118 continue
2119 rec_num = record.record_number
2120 if not isinstance(value, tuple):
2121 value = (value, )
2122 vindex = bisect_right(yo._values, value)
2123 yo._values.insert(vindex, value)
2124 yo._rec_by_val.insert(vindex, rec_num)
2125 yo._records[rec_num] = value
2126 table._indexen.add(yo)
2128 rec_num = record.record_number
2129 if rec_num in yo._records:
2130 value = yo._records[rec_num]
2131 vindex = bisect_left(yo._values, value)
2132 yo._values.pop(vindex)
2133 yo._rec_by_val.pop(vindex)
2134 value = yo.key(record)
2135 if value is DoNotIndex:
2136 return
2137 if not isinstance(value, tuple):
2138 value = (value, )
2139 vindex = bisect_right(yo._values, value)
2140 yo._values.insert(vindex, value)
2141 yo._rec_by_val.insert(vindex, rec_num)
2142 yo._records[rec_num] = value
2144 if isinstance(match, _DbfRecord):
2145 if match.record_table is yo._table:
2146 return match.record_number in yo._records
2147 match = yo.key(match)
2148 elif not isinstance(match, tuple):
2149 match = (match, )
2150 return yo.find(match) != -1
2152 if isinstance(key, int):
2153 count = len(yo._values)
2154 if not -count <= key < count:
2155 raise IndexError("Record %d is not in list." % key)
2156 rec_num = yo._rec_by_val[key]
2157 return yo._table.get_record(rec_num)
2158 elif isinstance(key, slice):
2159 result = List(field_names=yo._table.field_names)
2160 yo._table._dbflists.add(result)
2161 start, stop, step = key.start, key.stop, key.step
2162 if start is None: start = 0
2163 if stop is None: stop = len(yo._rec_by_val)
2164 if step is None: step = 1
2165 for loc in range(start, stop, step):
2166 record = yo._table.get_record(yo._rec_by_val[loc])
2167 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2168 result._current = 0 if result else -1
2169 return result
2170 elif isinstance (key, (str, unicode, tuple, _DbfRecord)):
2171 if isinstance(key, _DbfRecord):
2172 key = yo.key(key)
2173 elif not isinstance(key, tuple):
2174 key = (key, )
2175 loc = yo.find(key)
2176 if loc == -1:
2177 raise KeyError(key)
2178 return yo._table.get_record(yo._rec_by_val[loc])
2179 else:
2180 raise TypeError('indices must be integers, match objects must by strings or tuples')
2184 yo._table.close()
2185 yo._values[:] = []
2186 yo._rec_by_val[:] = []
2187 yo._records.clear()
2188 return False
2192 return len(yo._records)
2194 target = target[:len(match)]
2195 if isinstance(match[-1], (str, unicode)):
2196 target = list(target)
2197 target[-1] = target[-1][:len(match[-1])]
2198 target = tuple(target)
2199 return target == match
2201 value = yo._records.get(rec_num)
2202 if value is not None:
2203 vindex = bisect_left(yo._values, value)
2204 del yo._records[rec_num]
2205 yo._values.pop(vindex)
2206 yo._rec_by_val.pop(vindex)
2207 - def _search(yo, match, lo=0, hi=None):
2208 if hi is None:
2209 hi = len(yo._values)
2210 return bisect_left(yo._values, match, lo, hi)
2212 "removes all entries from index"
2213 yo._values[:] = []
2214 yo._rec_by_val[:] = []
2215 yo._records.clear()
2218 - def find(yo, match, partial=False):
2219 "returns numeric index of (partial) match, or -1"
2220 if isinstance(match, _DbfRecord):
2221 if match.record_number in yo._records:
2222 return yo._values.index(yo._records[match.record_number])
2223 else:
2224 return -1
2225 if not isinstance(match, tuple):
2226 match = (match, )
2227 loc = yo._search(match)
2228 while loc < len(yo._values) and yo._values[loc] == match:
2229 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
2230 loc += 1
2231 continue
2232 return loc
2233 if partial:
2234 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
2235 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
2236 loc += 1
2237 continue
2238 return loc
2239 return -1
2241 "returns numeric index of either (partial) match, or position of where match would be"
2242 if isinstance(match, _DbfRecord):
2243 if match.record_number in yo._records:
2244 return yo._values.index(yo._records[match.record_number])
2245 else:
2246 match = yo.key(match)
2247 if not isinstance(match, tuple):
2248 match = (match, )
2249 loc = yo._search(match)
2250 return loc
2251 @classmethod
2253
2254 def get_idx_records(data, length, howmany):
2255 ptr = 0
2256 current = 0
2257 while current < howmany:
2258 key = data[ptr:ptr+length].replace('\x00','')
2259 rec = io.unpackLongInt(data[ptr+length:ptr+length+4], bigendian=True)
2260 yield key, recnum
2261 ptr += length + 4
2262 current += 1
2263
2264 def next_item(idx_file, node_loc, keylen):
2265 idx_file.seek(node_loc)
2266 data_chunk = idx_file.read(512)
2267 attributes = io.unpackShortInt(data_chunk[:2])
2268 howmany = io.unpackShortInt(data_chunk[2:4])
2269 if attributes in (2, 3):
2270 for key, recnum in get_idx_records(data_chunk[12:512], keylen, howmany):
2271 yield key, recnum
2272 else:
2273 for ignore, next_node in get_idx_records(data_chunk[12:512], keylen, howmany):
2274 print ignore, next_node
2275 for key, recnum in next_item(idx_file, next_node, keylen):
2276 yield key, recnum
2277
2278
2279 idx = object.__new__(cls)
2280
2281 data = open(index_file, 'rb')
2282 header = data.read(512)
2283 rootnode = io.unpackLongInt(header[:4])
2284 keylen = io.unpackShortInt(header[12:14])
2285 idx.__doc__ = header[16:236].replace('\x00','')
2286 for_expr = header[236:456].replace('\x00','')
2287 if for_expr:
2288 idx.__doc__ += ' for ' + for_expr.replace('=','==')
2289 for rec in next_item(data, rootnode, keylen):
2290 print rec
2291
2292 - def index(yo, match, partial=False):
2293 "returns numeric index of (partial) match, or raises ValueError"
2294 loc = yo.find(match, partial)
2295 if loc == -1:
2296 if isinstance(match, _DbfRecord):
2297 raise ValueError("table <%s> record [%d] not in index <%s>" % (yo._table.filename, match.record_number, yo.__doc__))
2298 else:
2299 raise ValueError("match criteria <%s> not in index" % (match, ))
2300 return loc
2302 "reindexes all records"
2303 for record in yo._table:
2304 yo(record)
2305 - def query(yo, sql_command=None, python=None):
2306 """recognized sql commands are SELECT, UPDATE, REPLACE, INSERT, DELETE, and RECALL"""
2307 if sql_command:
2308 return sql(yo, sql_command)
2309 elif python is None:
2310 raise DbfError("query: python parameter must be specified")
2311 possible = List(desc="%s --> %s" % (yo._table.filename, python), field_names=yo._table.field_names)
2312 yo._table._dbflists.add(possible)
2313 query_result = {}
2314 select = 'query_result["keep"] = %s' % python
2315 g = {}
2316 for record in yo:
2317 query_result['keep'] = False
2318 g['query_result'] = query_result
2319 exec select in g, record
2320 if query_result['keep']:
2321 possible.append(record)
2322 record.write_record()
2323 return possible
2324 - def search(yo, match, partial=False):
2325 "returns dbf.List of all (partially) matching records"
2326 result = List(field_names=yo._table.field_names)
2327 yo._table._dbflists.add(result)
2328 if not isinstance(match, tuple):
2329 match = (match, )
2330 loc = yo._search(match)
2331 if loc == len(yo._values):
2332 return result
2333 while loc < len(yo._values) and yo._values[loc] == match:
2334 record = yo._table.get_record(yo._rec_by_val[loc])
2335 if not yo._table.use_deleted and record.has_been_deleted:
2336 loc += 1
2337 continue
2338 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2339 loc += 1
2340 if partial:
2341 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
2342 record = yo._table.get_record(yo._rec_by_val[loc])
2343 if not yo._table.use_deleted and record.has_been_deleted:
2344 loc += 1
2345 continue
2346 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2347 loc += 1
2348 return result
2349
2350 csv.register_dialect('dbf', DbfCsv)
2351
2352 sql_functions = {
2353 'select':None,
2354 'update':None,
2355 'insert':None,
2356 'delete':None,
2357 'count': None}
2359 "creates a function matching the sql criteria"
2360 function = """def func(records):
2361 \"\"\"%s\"\"\"
2362 matched = List(field_names=records[0].field_names)
2363 for rec in records:
2364 %s
2365
2366 if %s:
2367 matched.append(rec)
2368 return matched"""
2369 fields = []
2370 for field in records[0].field_names:
2371 if field in criteria:
2372 fields.append(field)
2373 fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields])
2374 g = {'List':List}
2375 function %= (criteria, fields, criteria)
2376
2377 exec function in g
2378 return g['func']
2379
2381 "creates a function matching to apply command to each record in records"
2382 function = """def func(records):
2383 \"\"\"%s\"\"\"
2384 changed = 0
2385 for rec in records:
2386 %s
2387
2388 %s
2389
2390 %s
2391 changed += rec.write_record()
2392 return changed"""
2393 fields = []
2394 for field in records[0].field_names:
2395 if field in command:
2396 fields.append(field)
2397 pre_fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields])
2398 post_fields = '\n '.join(['rec.%s = %s' % (field, field) for field in fields])
2399 g = dbf.sql_user_functions.copy()
2400 if '=' not in command and ' with ' in command.lower():
2401 offset = command.lower().index(' with ')
2402 command = command[:offset] + ' = ' + command[offset+6:]
2403 function %= (command, pre_fields, command, post_fields)
2404
2405 exec function in g
2406 return g['func']
2407
2408 -def sql(records, command):
2409 """recognized sql commands are SELECT, UPDATE, INSERT, DELETE, and RECALL"""
2410 table = records[0].record_table
2411 sql_command = command
2412 no_condition = False
2413 if ' for ' in command:
2414 command, condition = command.split(' for ')
2415 condition = sql_criteria(records, condition)
2416 else:
2417 def condition(records):
2418 return records[:]
2419 no_condition = True
2420 name, command = command.split(' ', 1)
2421 name = name.lower()
2422 field_names = table.field_names
2423 if name == 'select':
2424 if command.strip() != '*':
2425 field_names = command.replace(' ','').split(',')
2426 def command(records):
2427 return
2428 else:
2429 command = sql_cmd(records, command)
2430 if name not in ('delete','insert','recall','select','update','replace'):
2431 raise DbfError("unrecognized sql command: %s" % name.upper())
2432 if name == 'insert' and not no_condition:
2433 raise DbfError("FOR clause not allowed with INSERT")
2434 possible = List(desc=sql_command, field_names=field_names)
2435 tables = set()
2436 if name == 'insert':
2437 raise DbfError("INSERT not currently implemented")
2438 record = table.append()
2439 command(record)
2440 record.write_record()
2441 record.check_index()
2442 possible.append(record)
2443 changed = 0
2444 else:
2445 possible = condition(records)
2446 possible.field_names = field_names
2447 changed = command(possible)
2448 for record in possible:
2449 tables.add(record.record_table)
2450 if name == 'delete':
2451 record.delete_record()
2452 elif name == 'recall':
2453 record.undelete_record()
2454 elif name == 'select':
2455 pass
2456 elif name == 'update' or name == 'replace':
2457 pass
2458
2459 else:
2460 raise DbfError("unrecognized sql command: %s" % sql.upper)
2461 record.write_record()
2462 for list_table in tables:
2463 list_table._dbflists.add(possible)
2464 possible.modified = changed
2465 return possible
2467 "returns parameter unchanged"
2468 return value
2470 "ensures each tuple is the same length, using filler[-missing] for the gaps"
2471 final = []
2472 for t in tuples:
2473 if len(t) < length:
2474 final.append( tuple([item for item in t] + filler[len(t)-length:]) )
2475 else:
2476 final.append(t)
2477 return tuple(final)
2479 if cp not in code_pages:
2480 for code_page in sorted(code_pages.keys()):
2481 sd, ld = code_pages[code_page]
2482 if cp == sd or cp == ld:
2483 if sd is None:
2484 raise DbfError("Unsupported codepage: %s" % ld)
2485 cp = code_page
2486 break
2487 else:
2488 raise DbfError("Unsupported codepage: %s" % cp)
2489 sd, ld = code_pages[cp]
2490 return cp, sd, ld
2491 -def ascii(new_setting=None):
2498 -def codepage(cp=None):
2499 "get/set default codepage for any new tables"
2500 global default_codepage
2501 cp, sd, ld = _codepage_lookup(cp or default_codepage)
2502 default_codepage = sd
2503 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2511 version = 'dBase IV w/memos (non-functional)'
2512 _versionabbv = 'db4'
2513 _fieldtypes = {
2514 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
2515 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
2516 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
2517 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
2518 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
2519 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
2520 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
2521 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
2522 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
2523 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2524 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2525 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2526 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
2527 _memoext = '.dbt'
2528 _memotypes = ('G','M','P')
2529 _memoClass = _VfpMemo
2530 _yesMemoMask = '\x8b'
2531 _noMemoMask = '\x04'
2532 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
2533 _variable_fields = ('C','F','N')
2534 _character_fields = ('C','M')
2535 _decimal_fields = ('F','N')
2536 _numeric_fields = ('B','F','I','N','Y')
2537 _supported_tables = ('\x04', '\x8b')
2538 _dbfTableHeader = ['\x00'] * 32
2539 _dbfTableHeader[0] = '\x8b'
2540 _dbfTableHeader[10] = '\x01'
2541 _dbfTableHeader[29] = '\x03'
2542 _dbfTableHeader = ''.join(_dbfTableHeader)
2543 _dbfTableHeaderExtra = ''
2544 _use_deleted = True
2546 "dBase III specific"
2547 if yo._meta.header.version == '\x8b':
2548 try:
2549 yo._meta.memo = yo._memoClass(yo._meta)
2550 except:
2551 yo._meta.dfd.close()
2552 yo._meta.dfd = None
2553 raise
2554 if not yo._meta.ignorememos:
2555 for field in yo._meta.fields:
2556 if yo._meta[field]['type'] in yo._memotypes:
2557 if yo._meta.header.version != '\x8b':
2558 yo._meta.dfd.close()
2559 yo._meta.dfd = None
2560 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
2561 elif not os.path.exists(yo._meta.memoname):
2562 yo._meta.dfd.close()
2563 yo._meta.dfd = None
2564 raise DbfError("Table structure corrupt: memo fields exist without memo file")
2565 break
2566