1 "table definitions"
2 import os
3 import sys
4 import csv
5 import codecs
6 import locale
7 import unicodedata
8 import weakref
9 from array import array
10 from decimal import Decimal
11 from shutil import copyfileobj
12 from dbf import _io as io
13 from dbf.dates import Date, DateTime, Time
14 from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode
15
16 input_decoding = locale.getdefaultlocale()[1]
17 default_codepage = 'cp1252'
18 return_ascii = True
19
20 version_map = {
21 '\x02' : 'FoxBASE',
22 '\x03' : 'dBase III Plus',
23 '\x04' : 'dBase IV',
24 '\x05' : 'dBase V',
25 '\x30' : 'Visual FoxPro',
26 '\x31' : 'Visual FoxPro (auto increment field)',
27 '\x43' : 'dBase IV SQL',
28 '\x7b' : 'dBase IV w/memos',
29 '\x83' : 'dBase III Plus w/memos',
30 '\x8b' : 'dBase IV w/memos',
31 '\x8e' : 'dBase IV w/SQL table',
32 '\xf5' : 'FoxPro w/memos'}
33
34 code_pages = {
35 '\x00' : ('ascii', "plain ol' ascii"),
36 '\x01' : ('cp437', 'U.S. MS-DOS'),
37 '\x02' : ('cp850', 'International MS-DOS'),
38 '\x03' : ('cp1252', 'Windows ANSI'),
39 '\x04' : ('mac_roman', 'Standard Macintosh'),
40 '\x08' : ('cp865', 'Danish OEM'),
41 '\x09' : ('cp437', 'Dutch OEM'),
42 '\x0A' : ('cp850', 'Dutch OEM (secondary)'),
43 '\x0B' : ('cp437', 'Finnish OEM'),
44 '\x0D' : ('cp437', 'French OEM'),
45 '\x0E' : ('cp850', 'French OEM (secondary)'),
46 '\x0F' : ('cp437', 'German OEM'),
47 '\x10' : ('cp850', 'German OEM (secondary)'),
48 '\x11' : ('cp437', 'Italian OEM'),
49 '\x12' : ('cp850', 'Italian OEM (secondary)'),
50 '\x13' : ('cp932', 'Japanese Shift-JIS'),
51 '\x14' : ('cp850', 'Spanish OEM (secondary)'),
52 '\x15' : ('cp437', 'Swedish OEM'),
53 '\x16' : ('cp850', 'Swedish OEM (secondary)'),
54 '\x17' : ('cp865', 'Norwegian OEM'),
55 '\x18' : ('cp437', 'Spanish OEM'),
56 '\x19' : ('cp437', 'English OEM (Britain)'),
57 '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'),
58 '\x1B' : ('cp437', 'English OEM (U.S.)'),
59 '\x1C' : ('cp863', 'French OEM (Canada)'),
60 '\x1D' : ('cp850', 'French OEM (secondary)'),
61 '\x1F' : ('cp852', 'Czech OEM'),
62 '\x22' : ('cp852', 'Hungarian OEM'),
63 '\x23' : ('cp852', 'Polish OEM'),
64 '\x24' : ('cp860', 'Portugese OEM'),
65 '\x25' : ('cp850', 'Potugese OEM (secondary)'),
66 '\x26' : ('cp866', 'Russian OEM'),
67 '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'),
68 '\x40' : ('cp852', 'Romanian OEM'),
69 '\x4D' : ('cp936', 'Chinese GBK (PRC)'),
70 '\x4E' : ('cp949', 'Korean (ANSI/OEM)'),
71 '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'),
72 '\x50' : ('cp874', 'Thai (ANSI/OEM)'),
73 '\x57' : ('cp1252', 'ANSI'),
74 '\x58' : ('cp1252', 'Western European ANSI'),
75 '\x59' : ('cp1252', 'Spanish ANSI'),
76 '\x64' : ('cp852', 'Eastern European MS-DOS'),
77 '\x65' : ('cp866', 'Russian MS-DOS'),
78 '\x66' : ('cp865', 'Nordic MS-DOS'),
79 '\x67' : ('cp861', 'Icelandic MS-DOS'),
80 '\x68' : (None, 'Kamenicky (Czech) MS-DOS'),
81 '\x69' : (None, 'Mazovia (Polish) MS-DOS'),
82 '\x6a' : ('cp737', 'Greek MS-DOS (437G)'),
83 '\x6b' : ('cp857', 'Turkish MS-DOS'),
84 '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'),
85 '\x79' : ('cp949', 'Korean Windows'),
86 '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'),
87 '\x7b' : ('cp932', 'Japanese Windows'),
88 '\x7c' : ('cp874', 'Thai Windows'),
89 '\x7d' : ('cp1255', 'Hebrew Windows'),
90 '\x7e' : ('cp1256', 'Arabic Windows'),
91 '\xc8' : ('cp1250', 'Eastern European Windows'),
92 '\xc9' : ('cp1251', 'Russian Windows'),
93 '\xca' : ('cp1254', 'Turkish Windows'),
94 '\xcb' : ('cp1253', 'Greek Windows'),
95 '\x96' : ('mac_cyrillic', 'Russian Macintosh'),
96 '\x97' : ('mac_latin2', 'Macintosh EE'),
97 '\x98' : ('mac_greek', 'Greek Macintosh') }
98
99 if sys.version_info[:2] < (2, 6):
102 "Emulate PyProperty_Type() in Objects/descrobject.c"
103
104 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
105 self.fget = fget
106 self.fset = fset
107 self.fdel = fdel
108 self.__doc__ = doc or fget.__doc__
110 self.fget = func
111 if not self.__doc__:
112 self.__doc__ = fget.__doc__
113 - def __get__(self, obj, objtype=None):
114 if obj is None:
115 return self
116 if self.fget is None:
117 raise AttributeError, "unreadable attribute"
118 return self.fget(obj)
120 if self.fset is None:
121 raise AttributeError, "can't set attribute"
122 self.fset(obj, value)
124 if self.fdel is None:
125 raise AttributeError, "can't delete attribute"
126 self.fdel(obj)
128 self.fset = func
129 return self
131 self.fdel = func
132 return self
134 """Provides routines to extract and save data within the fields of a dbf record."""
135 __slots__ = ['_recnum', '_layout', '_data', '__weakref__']
137 """calls appropriate routine to fetch value stored in field from array
138 @param record_data: the data portion of the record
139 @type record_data: array of characters
140 @param fielddef: description of the field definition
141 @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags'
142 @returns: python data stored in field"""
143
144 field_type = fielddef['type']
145 retrieve = yo._layout.fieldtypes[field_type]['Retrieve']
146 datum = retrieve(record_data, fielddef, yo._layout.memo)
147 if field_type in yo._layout.character_fields:
148 datum = yo._layout.decoder(datum)[0]
149 if yo._layout.return_ascii:
150 try:
151 datum = yo._layout.output_encoder(datum)[0]
152 except UnicodeEncodeError:
153 datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore')
154 return datum
156 "calls appropriate routine to convert value to ascii bytes, and save it in record"
157 field_type = fielddef['type']
158 update = yo._layout.fieldtypes[field_type]['Update']
159 if field_type in yo._layout.character_fields:
160 if not isinstance(value, unicode):
161 if yo._layout.input_decoder is None:
162 raise NonUnicode("String not in unicode format, no default encoding specified")
163 value = yo._layout.input_decoder(value)[0]
164 value = yo._layout.encoder(value)[0]
165 bytes = array('c', update(value, fielddef, yo._layout.memo))
166 size = fielddef['length']
167 if len(bytes) > size:
168 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size))
169 blank = array('c', ' ' * size)
170 start = fielddef['start']
171 end = start + size
172 blank[:len(bytes)] = bytes[:]
173 yo._data[start:end] = blank[:]
174 yo._updateDisk(yo._recnum * yo._layout.header.record_length + yo._layout.header.start, yo._data.tostring())
186 results = []
187 if not specs:
188 specs = yo._layout.index
189 specs = _normalize_tuples(tuples=specs, length=2, filler=[_nop])
190 for field, func in specs:
191 results.append(func(yo[field]))
192 return tuple(results)
193
199 if name[0:2] == '__' and name[-2:] == '__':
200 raise AttributeError, 'Method %s is not implemented.' % name
201 elif not name in yo._layout.fields:
202 raise FieldMissing(name)
203 try:
204 fielddef = yo._layout[name]
205 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef)
206 return value
207 except DbfError, error:
208 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message)
209 raise
226 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
272 result = []
273 for field in yo.field_names:
274 result.append("%-10s: %s" % (field, yo[field]))
275 return '\n'.join(result)
277 return yo._data.tostring()
279 "creates a blank record data chunk"
280 layout = yo._layout
281 ondisk = layout.ondisk
282 layout.ondisk = False
283 yo._data = array('c', ' ' * layout.header.record_length)
284 layout.memofields = []
285 for field in layout.fields:
286 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']())
287 if layout[field]['type'] in layout.memotypes:
288 layout.memofields.append(field)
289 layout.blankrecord = yo._data[:]
290 layout.ondisk = ondisk
295 @property
300 "saves a dictionary into a records fields\nkeys with no matching field will raise a FieldMissing exception unless drop = True"
301 for key in dict:
302 if not key in yo.field_names:
303 if drop:
304 continue
305 raise FieldMissing(key)
306 yo.__setattr__(key, dict[key])
307 @property
309 "marked for deletion?"
310 return yo._data[0] == '*'
311 @property
313 "physical record number"
314 return yo._recnum
315 @property
317 table = yo._layout.table()
318 if table is None:
319 raise DbfError("table is no longer available")
320 return table
335 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty."
336 keys = yo._layout.fields
337 if blank:
338 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys]
339 else:
340 values = [yo[field] for field in keys]
341 return dict(zip(keys, values))
347 """Provides access to memo fields as dictionaries
348 must override _init, _get_memo, and _put_memo to
349 store memo contents to disk"""
351 "initialize disk file usage"
353 "retrieve memo contents from disk"
355 "store memo contents to disk"
357 ""
358 yo.meta = meta
359 yo.memory = {}
360 yo.nextmemo = 1
361 yo._init()
362 yo.meta.newmemofile = False
364 "gets the memo in block"
365 if yo.meta.ignorememos or not block:
366 return ''
367 if yo.meta.ondisk:
368 return yo._get_memo(block)
369 else:
370 return yo.memory[block]
372 "stores data in memo file, returns block number"
373 if yo.meta.ignorememos or data == '':
374 return 0
375 if yo.meta.inmemory:
376 thismemo = yo.nextmemo
377 yo.nextmemo += 1
378 yo.memory[thismemo] = data
379 else:
380 thismemo = yo._put_memo(data)
381 return thismemo
384 "dBase III specific"
385 yo.meta.memo_size= 512
386 yo.record_header_length = 2
387 if yo.meta.ondisk and not yo.meta.ignorememos:
388 if yo.meta.newmemofile:
389 yo.meta.mfd = open(yo.meta.memoname, 'w+b')
390 yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508)
391 else:
392 try:
393 yo.meta.mfd = open(yo.meta.memoname, 'r+b')
394 yo.meta.mfd.seek(0)
395 yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4))
396 except:
397 raise DbfError("memo file appears to be corrupt")
399 block = int(block)
400 yo.meta.mfd.seek(block * yo.meta.memo_size)
401 eom = -1
402 data = ''
403 while eom == -1:
404 newdata = yo.meta.mfd.read(yo.meta.memo_size)
405 if not newdata:
406 return data
407 data += newdata
408 eom = data.find('\x1a\x1a')
409 return data[:eom].rstrip()
411 length = len(data) + yo.record_header_length
412 blocks = length // yo.meta.memo_size
413 if length % yo.meta.memo_size:
414 blocks += 1
415 thismemo = yo.nextmemo
416 yo.nextmemo = thismemo + blocks
417 yo.meta.mfd.seek(0)
418 yo.meta.mfd.write(io.packLongInt(yo.nextmemo))
419 yo.meta.mfd.seek(thismemo * yo.meta.memo_size)
420 yo.meta.mfd.write(data)
421 yo.meta.mfd.write('\x1a\x1a')
422 if len(yo._get_memo(thismemo)) != len(data):
423 raise DbfError("unknown error: memo not saved")
424 return thismemo
427 "Visual Foxpro 6 specific"
428 if yo.meta.ondisk and not yo.meta.ignorememos:
429 yo.record_header_length = 8
430 if yo.meta.newmemofile:
431 if yo.meta.memo_size == 0:
432 yo.meta.memo_size = 1
433 elif 1 < yo.meta.memo_size < 33:
434 yo.meta.memo_size *= 512
435 yo.meta.mfd = open(yo.meta.memoname, 'w+b')
436 nextmemo = 512 // yo.meta.memo_size
437 if nextmemo * yo.meta.memo_size < 512:
438 nextmemo += 1
439 yo.nextmemo = nextmemo
440 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \
441 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504)
442 else:
443 try:
444 yo.meta.mfd = open(yo.meta.memoname, 'r+b')
445 yo.meta.mfd.seek(0)
446 header = yo.meta.mfd.read(512)
447 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True)
448 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True)
449 except:
450 raise DbfError("memo file appears to be corrupt")
452 yo.meta.mfd.seek(block * yo.meta.memo_size)
453 header = yo.meta.mfd.read(8)
454 length = io.unpackLongInt(header[4:], bigendian=True)
455 return yo.meta.mfd.read(length)
457 yo.meta.mfd.seek(0)
458 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True)
459 yo.meta.mfd.seek(0)
460 length = len(data) + yo.record_header_length
461 blocks = length // yo.meta.memo_size
462 if length % yo.meta.memo_size:
463 blocks += 1
464 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True))
465 yo.meta.mfd.seek(thismemo*yo.meta.memo_size)
466 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data)
467 return thismemo
469 """Provides a framework for dbf style tables."""
470 _version = 'basic memory table'
471 _versionabbv = 'dbf'
472 _fieldtypes = {
473 'D' : { 'Type':'Date', 'Init':io.addDate, 'Blank':Date.today, 'Retrieve':io.retrieveDate, 'Update':io.updateDate, },
474 'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool, 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, },
475 'M' : { 'Type':'Memo', 'Init':io.addMemo, 'Blank':str, 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, } }
476 _memoext = ''
477 _memotypes = tuple('M', )
478 _memoClass = _DbfMemo
479 _yesMemoMask = ''
480 _noMemoMask = ''
481 _fixed_fields = ('M','D','L')
482 _variable_fields = tuple()
483 _character_fields = tuple('M', )
484 _decimal_fields = tuple()
485 _numeric_fields = tuple()
486 _dbfTableHeader = array('c', '\x00' * 32)
487 _dbfTableHeader[0] = '\x00'
488 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
489 _dbfTableHeader[10] = '\x01'
490 _dbfTableHeader[29] = '\x00'
491 _dbfTableHeader = _dbfTableHeader.tostring()
492 _dbfTableHeaderExtra = ''
493 _supported_tables = []
494 _read_only = False
495 _meta_only = False
496 _use_deleted = True
497 _backed_up = False
515 if len(data) != 32:
516 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data))
517 yo._data = array('c', data + '\x0d')
519 "get/set code page of table"
520 if cp is None:
521 return yo._data[29]
522 else:
523 if cp not in code_pages:
524 for code_page in sorted(code_pages.keys()):
525 sd, ld = code_pages[code_page]
526 if cp == sd or cp == ld:
527 if sd is None:
528 raise DbfError("Unsupported codepage: %s" % ld)
529 cp = code_page
530 break
531 else:
532 raise DbfError("Unsupported codepage: %s" % cp)
533 yo._data[29] = cp
534 return cp
535 @property
541 @data.setter
543 if len(bytes) < 32:
544 raise DbfError("length for data of %d is less than 32" % len(bytes))
545 yo._data[:] = array('c', bytes)
546 @property
548 "extra dbf info (located after headers, before data records)"
549 fieldblock = yo._data[32:]
550 for i in range(len(fieldblock)//32+1):
551 cr = i * 32
552 if fieldblock[cr] == '\x0d':
553 break
554 else:
555 raise DbfError("corrupt field structure")
556 cr += 33
557 return yo._data[cr:].tostring()
558 @extra.setter
560 fieldblock = yo._data[32:]
561 for i in range(len(fieldblock)//32+1):
562 cr = i * 32
563 if fieldblock[cr] == '\x0d':
564 break
565 else:
566 raise DbfError("corrupt field structure")
567 cr += 33
568 yo._data[cr:] = array('c', data)
569 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
570 @property
572 "number of fields (read-only)"
573 fieldblock = yo._data[32:]
574 for i in range(len(fieldblock)//32+1):
575 cr = i * 32
576 if fieldblock[cr] == '\x0d':
577 break
578 else:
579 raise DbfError("corrupt field structure")
580 return len(fieldblock[:cr]) // 32
581 @property
583 "field block structure"
584 fieldblock = yo._data[32:]
585 for i in range(len(fieldblock)//32+1):
586 cr = i * 32
587 if fieldblock[cr] == '\x0d':
588 break
589 else:
590 raise DbfError("corrupt field structure")
591 return fieldblock[:cr].tostring()
592 @fields.setter
594 fieldblock = yo._data[32:]
595 for i in range(len(fieldblock)//32+1):
596 cr = i * 32
597 if fieldblock[cr] == '\x0d':
598 break
599 else:
600 raise DbfError("corrupt field structure")
601 cr += 32
602 fieldlen = len(block)
603 if fieldlen % 32 != 0:
604 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen)
605 yo._data[32:cr] = array('c', block)
606 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
607 fieldlen = fieldlen // 32
608 recordlen = 1
609 for i in range(fieldlen):
610 recordlen += ord(block[i*32+16])
611 yo._data[10:12] = array('c', io.packShortInt(recordlen))
612 @property
614 "number of records (maximum 16,777,215)"
615 return io.unpackLongInt(yo._data[4:8].tostring())
616 @record_count.setter
619 @property
621 "length of a record (read_only) (max of 65,535)"
622 return io.unpackShortInt(yo._data[10:12].tostring())
623 @property
625 "starting position of first record in file (must be within first 64K)"
626 return io.unpackShortInt(yo._data[8:10].tostring())
627 @start.setter
630 @property
632 "date of last table modification (read-only)"
633 return io.unpackDate(yo._data[1:4].tostring())
634 @property
636 "dbf version"
637 return yo._data[0]
638 @version.setter
642 "implements the weakref table for records"
644 yo._meta = meta
645 yo._weakref_list = [weakref.ref(lambda x: None)] * count
659 yo._weakref_list.append(weakref.ref(record))
661 "returns records using current index"
663 yo._table = table
664 yo._index = -1
665 yo._more_records = True
669 while yo._more_records:
670 yo._index += 1
671 if yo._index >= len(yo._table):
672 yo._more_records = False
673 continue
674 record = yo._table[yo._index]
675 if not yo._table.use_deleted and record.has_been_deleted:
676 continue
677 return record
678 else:
679 raise StopIteration
681 "constructs fieldblock for disk table"
682 fieldblock = array('c', '')
683 memo = False
684 yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask))
685 for field in yo._meta.fields:
686 if yo._meta.fields.count(field) > 1:
687 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)")
688 fielddef = array('c', '\x00' * 32)
689 fielddef[:11] = array('c', io.packStr(field))
690 fielddef[11] = yo._meta[field]['type']
691 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start']))
692 fielddef[16] = chr(yo._meta[field]['length'])
693 fielddef[17] = chr(yo._meta[field]['decimals'])
694 fielddef[18] = chr(yo._meta[field]['flags'])
695 fieldblock.extend(fielddef)
696 if yo._meta[field]['type'] in yo._meta.memotypes:
697 memo = True
698 yo._meta.header.fields = fieldblock.tostring()
699 if memo:
700 yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask))
701 if yo._meta.memo is None:
702 yo._meta.memo = yo._memoClass(yo._meta)
704 "dBase III specific"
705 if yo._meta.header.version == '\x83':
706 try:
707 yo._meta.memo = yo._memoClass(yo._meta)
708 except:
709 yo._meta.dfd.close()
710 yo._meta.dfd = None
711 raise
712 if not yo._meta.ignorememos:
713 for field in yo._meta.fields:
714 if yo._meta[field]['type'] in yo._memotypes:
715 if yo._meta.header.version != '\x83':
716 yo._meta.dfd.close()
717 yo._meta.dfd = None
718 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
719 elif not os.path.exists(yo._meta.memoname):
720 yo._meta.dfd.close()
721 yo._meta.dfd = None
722 raise DbfError("Table structure corrupt: memo fields exist without memo file")
723 break
725 "builds the FieldList of names, types, and descriptions from the disk file"
726 offset = 1
727 fieldsdef = yo._meta.header.fields
728 if len(fieldsdef) % 32 != 0:
729 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
730 if len(fieldsdef) // 32 != yo.field_count:
731 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
732 for i in range(yo.field_count):
733 fieldblock = fieldsdef[i*32:(i+1)*32]
734 name = io.unpackStr(fieldblock[:11])
735 type = fieldblock[11]
736 if not type in yo._meta.fieldtypes:
737 raise DbfError("Unknown field type: %s" % type)
738 start = offset
739 length = ord(fieldblock[16])
740 offset += length
741 end = start + length
742 decimals = ord(fieldblock[17])
743 flags = ord(fieldblock[18])
744 yo._meta.fields.append(name)
745 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
747 "Returns field information Name Type(Length[,Decimals])"
748 name = yo._meta.fields[i]
749 type = yo._meta[name]['type']
750 length = yo._meta[name]['length']
751 decimals = yo._meta[name]['decimals']
752 if type in yo._decimal_fields:
753 description = "%s %s(%d,%d)" % (name, type, length, decimals)
754 elif type in yo._fixed_fields:
755 description = "%s %s" % (name, type)
756 else:
757 description = "%s %s(%d)" % (name, type, length)
758 return description
760 "loads the records from disk to memory"
761 if yo._meta_only:
762 raise DbfError("%s has been closed, records are unavailable" % yo.filename)
763 dfd = yo._meta.dfd
764 header = yo._meta.header
765 dfd.seek(header.start)
766 allrecords = dfd.read()
767 dfd.seek(0)
768 length = header.record_length
769 for i in range(header.record_count):
770 record_data = allrecords[length*i:length*i+length]
771 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True))
772 yo._index.append(i)
773 dfd.seek(0)
775 if specs is None:
776 specs = yo.field_names
777 elif isinstance(specs, str):
778 specs = specs.split(sep)
779 else:
780 specs = list(specs)
781 specs = [s.strip() for s in specs]
782 return specs
784 "synchronizes the disk file with current data"
785 if yo._meta.inmemory:
786 return
787 fd = yo._meta.dfd
788 fd.seek(0)
789 fd.write(yo._meta.header.data)
790 if not headeronly:
791 for record in yo._table:
792 record._updateDisk()
793 fd.flush()
794 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length)
802 if name in ('_index','_table'):
803 if yo._meta.ondisk:
804 yo._table = yo._Table(len(yo), yo._meta)
805 yo._index = range(len(yo))
806 else:
807 yo._table = []
808 yo._index = []
809 yo._loadtable()
810 return object.__getattribute__(yo, name)
812 if type(value) == int:
813 if not -yo._meta.header.record_count <= value < yo._meta.header.record_count:
814 raise IndexError("Record %d is not in table." % value)
815 return yo._table[yo._index[value]]
816 elif type(value) == slice:
817 sequence = DbfList(desc='%s --> %s' % (yo.filename, value))
818 for index in yo._index[value]:
819 record = yo._table[index]
820 if yo.use_deleted is True or not record.has_been_deleted:
821 sequence.append(record)
822 return sequence
823 else:
824 raise TypeError('type <%s> not valid for indexing' % type(value))
825 - def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False,
826 read_only=False, keep_memos=False, meta_only=False, codepage=None):
827 """open/create dbf file
828 filename should include path if needed
829 field_specs can be either a ;-delimited string or a list of strings
830 memo_size is always 512 for db3 memos
831 ignore_memos is useful if the memo file is missing or corrupt
832 read_only will load records into memory, then close the disk file
833 keep_memos will also load any memo fields into memory
834 meta_only will ignore all records, keeping only basic table information
835 codepage will override whatever is set in the table itself"""
836 if filename == ':memory:':
837 if field_specs is None:
838 raise DbfError("field list must be specified for in-memory tables")
839 elif type(yo) is DbfTable:
840 raise DbfError("only memory tables supported")
841 yo._meta = meta = yo._MetaData()
842 meta.table = weakref.ref(yo)
843 meta.filename = filename
844 meta.fields = []
845 meta.fieldtypes = yo._fieldtypes
846 meta.fixed_fields = yo._fixed_fields
847 meta.variable_fields = yo._variable_fields
848 meta.character_fields = yo._character_fields
849 meta.decimal_fields = yo._decimal_fields
850 meta.numeric_fields = yo._numeric_fields
851 meta.memotypes = yo._memotypes
852 meta.ignorememos = ignore_memos
853 meta.memo_size = memo_size
854 meta.input_decoder = codecs.getdecoder(input_decoding)
855 meta.output_encoder = codecs.getencoder(input_decoding)
856 meta.return_ascii = return_ascii
857 meta.header = header = yo._TableHeader(yo._dbfTableHeader)
858 header.extra = yo._dbfTableHeaderExtra
859 header.data
860 yo.codepage = codepage or default_codepage
861 if filename == ':memory:':
862 yo._index = []
863 yo._table = []
864 meta.ondisk = False
865 meta.inmemory = True
866 meta.memoname = ':memory:'
867 else:
868 base, ext = os.path.splitext(filename)
869 if ext == '':
870 meta.filename = base + '.dbf'
871 meta.memoname = base + yo._memoext
872 meta.ondisk = True
873 meta.inmemory = False
874 if field_specs:
875 if meta.ondisk:
876 meta.dfd = open(meta.filename, 'w+b')
877 meta.newmemofile = True
878 yo.add_fields(field_specs)
879 return
880 dfd = meta.dfd = open(meta.filename, 'r+b')
881 dfd.seek(0)
882 meta.header = header = yo._TableHeader(dfd.read(32))
883 if not header.version in yo._supported_tables:
884 dfd.close()
885 dfd = None
886 raise TypeError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version)))
887 yo.codepage = meta.header.codepage()
888 fieldblock = dfd.read(header.start - 32)
889 for i in range(len(fieldblock)//32+1):
890 fieldend = i * 32
891 if fieldblock[fieldend] == '\x0d':
892 break
893 else:
894 raise DbfError("corrupt field structure in header")
895 if len(fieldblock[:fieldend]) % 32 != 0:
896 raise DbfError("corrupt field structure in header")
897 header.fields = fieldblock[:fieldend]
898 header.extra = fieldblock[fieldend+1:]
899 yo._initializeFields()
900 yo._checkMemoIntegrity()
901 meta.current = -1
902 if len(yo) > 0:
903 meta.current = 0
904 dfd.seek(0)
905 if meta_only:
906 yo.close(keep_table=False, keep_memos=False)
907 elif read_only:
908 yo.close(keep_table=True, keep_memos=keep_memos)
916 if yo._read_only:
917 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename
918 elif yo._meta_only:
919 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename
920 else:
921 return __name__ + ".Table('%s')" % yo._meta.filename
923 if yo._read_only:
924 status = "read-only"
925 elif yo._meta_only:
926 status = "meta-only"
927 else:
928 status = "read/write"
929 str = """
930 Table: %s
931 Type: %s
932 Codepage: %s
933 Status: %s
934 Last updated: %s
935 Record count: %d
936 Field count: %d
937 Record length: %d
938 """ % (yo.filename, version_map.get(yo._meta.header.version, 'unknown - ' + hex(ord(yo._meta.header.version))),
939 yo.codepage, status, yo.last_update, len(yo), yo.field_count, yo.record_length)
940 str += "\n --Fields--\n"
941 for i in range(len(yo._meta.fields)):
942 str += " " + yo._fieldLayout(i) + "\n"
943 return str
944 @property
946 return "%s (%s)" % code_pages[yo._meta.header.codepage()]
947 @codepage.setter
948 - def codepage(yo, cp):
949 result = yo._meta.header.codepage(cp)
950 yo._meta.decoder = codecs.getdecoder(code_pages[result][0])
951 yo._meta.encoder = codecs.getencoder(code_pages[result][0])
952 @property
954 "the number of fields in the table"
955 return yo._meta.header.field_count
956 @property
958 "a list of the fields in the table"
959 return yo._meta.fields[:]
960 @property
962 "table's file name, including path (if specified on open)"
963 return yo._meta.filename
964 @property
966 "date of last update"
967 return yo._meta.header.update
968 @property
970 "table's memo name (if path included in filename on open)"
971 return yo._meta.memoname
972 @property
974 "number of bytes in a record"
975 return yo._meta.header.record_length
976 @property
978 "index number of the current record"
979 return yo._meta.current
980 @property
984 @property
986 "process or ignore deleted records"
987 return yo._use_deleted
988 @use_deleted.setter
991 @property
993 "returns the dbf type of the table"
994 return yo._version
996 """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]]
997 backup table is created with _backup appended to name
998 then modifies current structure"""
999 all_records = [record for record in yo]
1000 if yo:
1001 yo.create_backup()
1002 yo._meta.blankrecord = None
1003 meta = yo._meta
1004 offset = meta.header.record_length
1005 fields = yo._list_fields(field_specs, sep=';')
1006 for field in fields:
1007 try:
1008 name, format = field.split()
1009 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum():
1010 raise DbfError("Field names cannot start with _ or digits, and can only contain the _, letters, and digits")
1011 name = name.lower()
1012 if name in meta.fields:
1013 raise DbfError("Field '%s' already exists" % name)
1014 field_type = format[0].upper()
1015 if len(name) > 10:
1016 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name)))
1017 if not field_type in meta.fieldtypes.keys():
1018 raise DbfError("Unknown field type: %s" % field_type)
1019 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format)
1020 except ValueError:
1021 raise DbfError("invalid field specifier: %s" % field)
1022 start = offset
1023 end = offset + length
1024 offset = end
1025 meta.fields.append(name)
1026 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0}
1027 if meta[name]['type'] in yo._memotypes and meta.memo is None:
1028 meta.memo = yo._memoClass(meta)
1029 for record in yo:
1030 record[name] = meta.fieldtypes[field_type]['Blank']()
1031 yo._buildHeaderFields()
1032 yo._updateDisk()
1033 - def append(yo, kamikaze='', drop=False, multiple=1):
1034 "adds <multiple> blank records, and fills fields with dict/tuple values if present"
1035 if not yo.field_count:
1036 raise DbfError("No fields defined, cannot append")
1037 empty_table = len(yo) == 0
1038 dictdata = False
1039 tupledata = False
1040 if not isinstance(kamikaze, _DbfRecord):
1041 if isinstance(kamikaze, dict):
1042 dictdata = kamikaze
1043 kamikaze = ''
1044 elif isinstance(kamikaze, tuple):
1045 tupledata = kamikaze
1046 kamikaze = ''
1047 newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze)
1048 yo._table.append(newrecord)
1049 yo._index.append(yo._meta.header.record_count)
1050 yo._meta.header.record_count += 1
1051 if dictdata:
1052 newrecord.gather_fields(dictdata, drop)
1053 elif tupledata:
1054 for index, item in enumerate(tupledata):
1055 newrecord[index] = item
1056 elif kamikaze == str:
1057 for field in yo._meta.memofields:
1058 newrecord[field] = ''
1059 elif kamikaze:
1060 for field in yo._meta.memofields:
1061 newrecord[field] = kamikaze[field]
1062 multiple -= 1
1063 if multiple:
1064 data = newrecord._data
1065 single = yo._meta.header.record_count
1066 total = single + multiple
1067 while single < total:
1068 multi_record = _DbfRecord(single, yo._meta, kamikaze=data)
1069 yo._table.append(multi_record)
1070 yo._index.append(single)
1071 for field in yo._meta.memofields:
1072 multi_record[field] = newrecord[field]
1073 single += 1
1074 yo._meta.header.record_count = total
1075 yo._meta.current = yo._meta.header.record_count - 1
1076 newrecord = multi_record
1077 yo._updateDisk(headeronly=True)
1078 if empty_table:
1079 yo._meta.current = 0
1080 return newrecord
1082 "moves record pointer to previous usable record; returns True if no more usable records"
1083 while yo._meta.current > 0:
1084 yo._meta.current -= 1
1085 if yo.use_deleted or not yo.current().has_been_deleted:
1086 break
1087 else:
1088 yo._meta.current = -1
1089 return True
1090 return False
1091 - def bottom(yo, get_record=False):
1092 """sets record pointer to bottom of table
1093 if get_record, seeks to and returns last (non-deleted) record
1094 DbfError if table is empty
1095 Bof if all records deleted and use_deleted is False"""
1096 yo._meta.current = yo._meta.header.record_count
1097 if get_record:
1098 try:
1099 return yo.prev()
1100 except Bof:
1101 yo._meta.current = yo._meta.header.record_count
1102 raise Eof()
1103 - def close(yo, keep_table=False, keep_memos=False):
1104 """closes disk files
1105 ensures table data is available if keep_table
1106 ensures memo data is available if keep_memos"""
1107 if keep_table:
1108 yo._table
1109 else:
1110 if '_index' in dir(yo):
1111 del yo._table
1112 del yo._index
1113 yo._meta.inmemory = True
1114 if yo._meta.ondisk:
1115 yo._meta.dfd.close()
1116 yo._meta.dfd = None
1117 if '_index' in dir(yo):
1118 yo._read_only = True
1119 else:
1120 yo._meta_only = True
1121 if yo._meta.mfd is not None:
1122 if not keep_memos:
1123 yo._meta.ignorememos = True
1124 else:
1125 memo_fields = []
1126 for field in yo.field_names:
1127 if yo.is_memotype(field):
1128 memo_fields.append(field)
1129 for record in yo:
1130 for field in memo_fields:
1131 record[field] = record[field]
1132 yo._meta.mfd.close()
1133 yo._meta.mfd = None
1134 yo._meta.ondisk = False
1136 "creates a backup table -- ignored if memory table"
1137 if yo.filename.startswith(':memory:'):
1138 return
1139 if new_name is None:
1140 new_name = os.path.splitext(yo.filename)[0] + '_backup.dbf'
1141 else:
1142 overwrite = True
1143 if overwrite or not yo._backed_up:
1144 bkup = open(new_name, 'wb')
1145 try:
1146 yo._meta.dfd.seek(0)
1147 copyfileobj(yo._meta.dfd, bkup)
1148 yo._backed_up = True
1149 finally:
1150 bkup.close()
1152 "returns current logical record, or its index"
1153 if yo._meta.current < 0:
1154 raise Bof()
1155 elif yo._meta.current >= yo._meta.header.record_count:
1156 raise Eof()
1157 if index:
1158 return yo._meta.current
1159 return yo._table[yo._index[yo._meta.current]]
1161 """removes field(s) from the table
1162 creates backup files with _backup appended to the file name,
1163 then modifies current structure"""
1164 doomed = yo._list_fields(doomed)
1165 for victim in doomed:
1166 if victim not in yo._meta.fields:
1167 raise DbfError("field %s not in table -- delete aborted" % victim)
1168 all_records = [record for record in yo]
1169 yo.create_backup()
1170 for victim in doomed:
1171 yo._meta.fields.pop(yo._meta.fields.index(victim))
1172 start = yo._meta[victim]['start']
1173 end = yo._meta[victim]['end']
1174 for record in yo:
1175 record._data = record._data[:start] + record._data[end:]
1176 for field in yo._meta.fields:
1177 if yo._meta[field]['start'] == end:
1178 end = yo._meta[field]['end']
1179 yo._meta[field]['start'] = start
1180 yo._meta[field]['end'] = start + yo._meta[field]['length']
1181 start = yo._meta[field]['end']
1182 yo._buildHeaderFields()
1183 yo._updateDisk()
1194 - def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
1195 """writes the table using CSV or tab-delimited format, using the filename
1196 given if specified, otherwise the table name"""
1197 if filename is None:
1198 filename = yo.filename
1199 field_specs = yo._list_fields(field_specs)
1200 if records is None:
1201 records = yo
1202 format = format.lower()
1203 if format not in ('csv', 'tab'):
1204 raise DbfError("export format: csv or tab, not %s" % format)
1205 base, ext = os.path.splitext(filename)
1206 if ext.lower() in ('', '.dbf'):
1207 filename = base + "." + format
1208 fd = open(filename, 'wb')
1209 try:
1210 if format == 'csv':
1211 csvfile = csv.writer(fd, dialect='dbf')
1212 if header:
1213 csvfile.writerow(field_specs)
1214 for record in records:
1215 fields = []
1216 for fieldname in field_specs:
1217 fields.append(record[fieldname])
1218 csvfile.writerow(fields)
1219 else:
1220 if header:
1221 fd.write('\t'.join(field_specs) + '\n')
1222 for record in records:
1223 fields = []
1224 for fieldname in field_specs:
1225 fields.append(str(record[fieldname]))
1226 fd.write('\t'.join(fields) + '\n')
1227 finally:
1228 fd.close()
1229 fd = None
1230 return len(records)
1232 "returns record at physical_index[recno]"
1233 return yo._table[recno]
1234 - def goto(yo, criteria):
1235 """changes the record pointer to the first matching (non-deleted) record
1236 criteria should be either a tuple of tuple(value, field, func) triples,
1237 or an integer to go to"""
1238 if isinstance(criteria, int):
1239 if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count:
1240 raise IndexError("Record %d does not exist" % criteria)
1241 if criteria < 0:
1242 criteria += yo._meta.header.record_count
1243 yo._meta.current = criteria
1244 return yo.current()
1245 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop])
1246 specs = tuple([(field, func) for value, field, func in criteria])
1247 match = tuple([value for value, field, func in criteria])
1248 current = yo.current(index=True)
1249 matchlen = len(match)
1250 while not yo.Eof():
1251 record = yo.current()
1252 results = record(*specs)
1253 if results == match:
1254 return record
1255 return yo.goto(current)
1256 - def index(yo, sort=None, reverse=False):
1277 "returns True if name is a memo type field"
1278 return yo._meta[name]['type'] in yo._memotypes
1279 - def new(yo, filename, _field_specs=None):
1280 "returns a new table of the same type"
1281 if _field_specs is None:
1282 _field_specs = yo.structure()
1283 if filename != ':memory:':
1284 path, name = os.path.split(filename)
1285 if path == "":
1286 filename = os.path.join(os.path.split(yo.filename)[0], filename)
1287 elif name == "":
1288 filename = os.path.join(path, os.path.split(yo.filename)[1])
1289 return yo.__class__(filename, _field_specs)
1291 "set record pointer to next (non-deleted) record, and return it"
1292 if yo.eof():
1293 raise Eof()
1294 return yo.current()
1295 - def pack(yo, _pack=True):
1296 "physically removes all deleted records"
1297 newtable = []
1298 newindex = []
1299 i = 0
1300 for record in yo._table:
1301 if record.has_been_deleted and _pack:
1302 record._recnum = -1
1303 else:
1304 record._recnum = i
1305 newtable.append(record)
1306 newindex.append(i)
1307 i += 1
1308 yo._table = newtable
1309 yo._index = newindex
1310 yo._meta.header.record_count = i
1311 yo._current = -1
1312 yo._meta.index = ''
1313 yo._updateDisk()
1315 "set record pointer to previous (non-deleted) record, and return it"
1316 if yo.bof():
1317 raise Bof
1318 return yo.current()
1319 - def query(yo, sql=None, python=None):
1320 "uses exec to perform python queries on the table"
1321 if python is None:
1322 raise DbfError("query: python parameter must be specified")
1323 possible = DbfList(desc="%s --> %s" % (yo.filename, python))
1324 query_result = {}
1325 select = 'query_result["keep"] = %s' % python
1326 g = {}
1327 for record in yo:
1328 query_result['keep'] = False
1329 g['query_result'] = query_result
1330 exec select in g, record
1331 if query_result['keep']:
1332 possible.append(record)
1333 return possible
1335 "renames an existing field"
1336 if yo:
1337 yo.create_backup()
1338 if not oldname in yo._meta.fields:
1339 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname)
1340 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum():
1341 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits")
1342 newname = newname.lower()
1343 if newname in yo._meta.fields:
1344 raise DbfError("field --%s-- already exists" % newname)
1345 if len(newname) > 10:
1346 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname)))
1347 yo._meta[newname] = yo._meta[oldname]
1348 yo._meta.fields[yo._meta.fields.index(oldname)] = newname
1349 yo._buildHeaderFields()
1350 yo._updateDisk(headeronly=True)
1351 - def search(yo, match, fuzzy=None, indices=False):
1352 """searches using a binary algorythm
1353 looking for records that match the criteria in match, which is a tuple
1354 with a data item per ordered field. table must be sorted. if index,
1355 returns a list of records' indices from the current sort order.
1356 """
1357 if yo._meta.index is None:
1358 raise DbfError('table must be indexed to use Search')
1359 matchlen = len(match)
1360 if fuzzy:
1361 matchlen -= 1
1362 fuzzy_match = match[-1]
1363 fuzzy_field = yo._meta.index[matchlen][0]
1364 match = match[:-1]
1365 records = DbfList(desc="%s --> search: index=%s, match=%s, fuzzy=%s(%s))" % (yo.filename, yo.index(), match, fuzzy.__name__, fuzzy_match))
1366 else:
1367 records = DbfList(desc="%s --> search: index=%s, match=%s)" % (yo.filename, yo.index(), match))
1368 if indices:
1369 records = []
1370 if not isinstance(match, tuple):
1371 match = tuple(match)
1372 segment = len(yo)
1373 current = 0
1374 toosoon = True
1375 notFound = True
1376 while notFound:
1377 segment = segment // 2
1378 if toosoon:
1379 current += segment
1380 else:
1381 current -= segment
1382 if current % 2:
1383 segment += 1
1384 if current == len(yo) or segment == 0:
1385 break
1386 value = yo._meta.orderresults[yo[current].record_number][:matchlen]
1387 if value < match:
1388 toosoon = True
1389 elif value > match:
1390 toosoon = False
1391 else:
1392 notFound = False
1393 break
1394 if current == 0:
1395 break
1396 if notFound:
1397 return records
1398 while current > 0:
1399 current -= 1
1400 value = yo._meta.orderresults[yo[current].record_number][:matchlen]
1401 if value != match:
1402 current += 1
1403 break
1404 while True:
1405 value = yo._meta.orderresults[yo[current].record_number][:matchlen]
1406 if value != match:
1407 break
1408 if yo.use_deleted or not yo[current].has_been_deleted:
1409 if indices:
1410 records.append(current)
1411 else:
1412 records.append(yo[current])
1413 current += 1
1414 if current == len(yo):
1415 break
1416 if fuzzy:
1417 if indices:
1418 records = [rec for rec in records if fuzzy(yo[rec][fuzzy_field]) == fuzzy_match]
1419 else:
1420 final_records = [rec for rec in records if fuzzy(rec[fuzzy_field]) == fuzzy_match]
1421 records.clear()
1422 records.extend(final_records)
1423 return records
1424 - def size(yo, field):
1425 "returns size of field as a tuple of (length, decimals)"
1426 if field in yo:
1427 return (yo._meta[field]['length'], yo._meta[field]['decimals'])
1428 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1430 """return list of fields suitable for creating same table layout
1431 @param fields: list of fields or None for all fields"""
1432 field_specs = []
1433 fields = yo._list_fields(fields)
1434 try:
1435 for name in fields:
1436 field_specs.append(yo._fieldLayout(yo.field_names.index(name)))
1437 except ValueError:
1438 raise DbfError("field --%s-- does not exist" % name)
1439 return field_specs
1440 - def top(yo, get_record=False):
1441 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record
1442 DbfError if table is empty
1443 Eof if all records are deleted and use_deleted is False"""
1444 yo._meta.current = -1
1445 if get_record:
1446 try:
1447 return yo.next()
1448 except Eof:
1449 yo._meta.current = -1
1450 raise Bof()
1451 - def type(yo, field):
1452 "returns type of field"
1453 if field in yo:
1454 return yo._meta[field]['type']
1455 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1456 - def zap(yo, areyousure=False):
1457 """removes all records from table -- this cannot be undone!
1458 areyousure must be True, else error is raised"""
1459 if areyousure:
1460 yo._table = []
1461 yo._index = []
1462 yo._meta.header.record_count = 0
1463 yo._current = -1
1464 yo._meta.index = ''
1465 yo._updateDisk()
1466 else:
1467 raise DbfError("You must say you are sure to wipe the table")
1468
1470 """Provides an interface for working with dBase III tables."""
1471 _version = 'dBase III Plus'
1472 _versionabbv = 'db3'
1473 _fieldtypes = {
1474 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
1475 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
1476 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
1477 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1478 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric} }
1479 _memoext = '.dbt'
1480 _memotypes = ('M',)
1481 _memoClass = _Db3Memo
1482 _yesMemoMask = '\x80'
1483 _noMemoMask = '\x7f'
1484 _fixed_fields = ('D','L','M')
1485 _variable_fields = ('C','N')
1486 _character_fields = ('C','M')
1487 _decimal_fields = ('N',)
1488 _numeric_fields = ('N',)
1489 _dbfTableHeader = array('c', '\x00' * 32)
1490 _dbfTableHeader[0] = '\x03'
1491 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
1492 _dbfTableHeader[10] = '\x01'
1493 _dbfTableHeader[29] = '\x03'
1494 _dbfTableHeader = _dbfTableHeader.tostring()
1495 _dbfTableHeaderExtra = ''
1496 _supported_tables = ['\x03', '\x83']
1497 _read_only = False
1498 _meta_only = False
1499 _use_deleted = True
1501 "dBase III specific"
1502 if yo._meta.header.version == '\x83':
1503 try:
1504 yo._meta.memo = yo._memoClass(yo._meta)
1505 except:
1506 yo._meta.dfd.close()
1507 yo._meta.dfd = None
1508 raise
1509 if not yo._meta.ignorememos:
1510 for field in yo._meta.fields:
1511 if yo._meta[field]['type'] in yo._memotypes:
1512 if yo._meta.header.version != '\x83':
1513 yo._meta.dfd.close()
1514 yo._meta.dfd = None
1515 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
1516 elif not os.path.exists(yo._meta.memoname):
1517 yo._meta.dfd.close()
1518 yo._meta.dfd = None
1519 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1520 break
1522 "builds the FieldList of names, types, and descriptions"
1523 offset = 1
1524 fieldsdef = yo._meta.header.fields
1525 if len(fieldsdef) % 32 != 0:
1526 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1527 if len(fieldsdef) // 32 != yo.field_count:
1528 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1529 for i in range(yo.field_count):
1530 fieldblock = fieldsdef[i*32:(i+1)*32]
1531 name = io.unpackStr(fieldblock[:11])
1532 type = fieldblock[11]
1533 if not type in yo._meta.fieldtypes:
1534 raise DbfError("Unknown field type: %s" % type)
1535 start = offset
1536 length = ord(fieldblock[16])
1537 offset += length
1538 end = start + length
1539 decimals = ord(fieldblock[17])
1540 flags = ord(fieldblock[18])
1541 yo._meta.fields.append(name)
1542 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1544 'Provides an interface for working with FoxPro 2 tables'
1545 _version = 'Foxpro'
1546 _versionabbv = 'fp'
1547 _fieldtypes = {
1548 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
1549 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
1550 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
1551 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
1552 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
1553 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo},
1554 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1555 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1556 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
1557 _memoext = '.fpt'
1558 _memotypes = ('G','M','P')
1559 _memoClass = _VfpMemo
1560 _yesMemoMask = '\xf5'
1561 _noMemoMask = '\x03'
1562 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1563 _variable_fields = ('C','F','N')
1564 _character_fields = ('C','M')
1565 _decimal_fields = ('F','N')
1566 _numeric_fields = ('B','F','I','N','Y')
1567 _supported_tables = ('\x03', '\xf5')
1568 _dbfTableHeader = array('c', '\x00' * 32)
1569 _dbfTableHeader[0] = '\x30'
1570 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1571 _dbfTableHeader[10] = '\x01'
1572 _dbfTableHeader[29] = '\x03'
1573 _dbfTableHeader = _dbfTableHeader.tostring()
1574 _dbfTableHeaderExtra = '\x00' * 263
1575 _use_deleted = True
1577 if os.path.exists(yo._meta.memoname):
1578 try:
1579 yo._meta.memo = yo._memoClass(yo._meta)
1580 except:
1581 yo._meta.dfd.close()
1582 yo._meta.dfd = None
1583 raise
1584 if not yo._meta.ignorememos:
1585 for field in yo._meta.fields:
1586 if yo._meta[field]['type'] in yo._memotypes:
1587 if not os.path.exists(yo._meta.memoname):
1588 yo._meta.dfd.close()
1589 yo._meta.dfd = None
1590 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1591 break
1593 "builds the FieldList of names, types, and descriptions"
1594 offset = 1
1595 fieldsdef = yo._meta.header.fields
1596 if len(fieldsdef) % 32 != 0:
1597 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1598 if len(fieldsdef) // 32 != yo.field_count:
1599 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1600 for i in range(yo.field_count):
1601 fieldblock = fieldsdef[i*32:(i+1)*32]
1602 name = io.unpackStr(fieldblock[:11])
1603 type = fieldblock[11]
1604 if not type in yo._meta.fieldtypes:
1605 raise DbfError("Unknown field type: %s" % type)
1606 elif type == '0':
1607 return
1608 start = offset
1609 length = ord(fieldblock[16])
1610 offset += length
1611 end = start + length
1612 decimals = ord(fieldblock[17])
1613 flags = ord(fieldblock[18])
1614 yo._meta.fields.append(name)
1615 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1616
1618 'Provides an interface for working with Visual FoxPro 6 tables'
1619 _version = 'Visual Foxpro v6'
1620 _versionabbv = 'vfp'
1621 _fieldtypes = {
1622 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
1623 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
1624 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
1625 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
1626 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
1627 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
1628 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
1629 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
1630 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
1631 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
1632 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
1633 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
1634 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
1635 _memoext = '.fpt'
1636 _memotypes = ('G','M','P')
1637 _memoClass = _VfpMemo
1638 _yesMemoMask = '\x30'
1639 _noMemoMask = '\x30'
1640 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1641 _variable_fields = ('C','F','N')
1642 _character_fields = ('C','M')
1643 _decimal_fields = ('F','N')
1644 _numeric_fields = ('B','F','I','N','Y')
1645 _supported_tables = ('\x30',)
1646 _dbfTableHeader = array('c', '\x00' * 32)
1647 _dbfTableHeader[0] = '\x30'
1648 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1649 _dbfTableHeader[10] = '\x01'
1650 _dbfTableHeader[29] = '\x03'
1651 _dbfTableHeader = _dbfTableHeader.tostring()
1652 _dbfTableHeaderExtra = '\x00' * 263
1653 _use_deleted = True
1655 if os.path.exists(yo._meta.memoname):
1656 try:
1657 yo._meta.memo = yo._memoClass(yo._meta)
1658 except:
1659 yo._meta.dfd.close()
1660 yo._meta.dfd = None
1661 raise
1662 if not yo._meta.ignorememos:
1663 for field in yo._meta.fields:
1664 if yo._meta[field]['type'] in yo._memotypes:
1665 if not os.path.exists(yo._meta.memoname):
1666 yo._meta.dfd.close()
1667 yo._meta.dfd = None
1668 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1669 break
1671 "builds the FieldList of names, types, and descriptions"
1672 offset = 1
1673 fieldsdef = yo._meta.header.fields
1674 for i in range(yo.field_count):
1675 fieldblock = fieldsdef[i*32:(i+1)*32]
1676 name = io.unpackStr(fieldblock[:11])
1677 type = fieldblock[11]
1678 if not type in yo._meta.fieldtypes:
1679 raise DbfError("Unknown field type: %s" % type)
1680 elif type == '0':
1681 return
1682 start = io.unpackLongInt(fieldblock[12:16])
1683 length = ord(fieldblock[16])
1684 offset += length
1685 end = start + length
1686 decimals = ord(fieldblock[17])
1687 flags = ord(fieldblock[18])
1688 yo._meta.fields.append(name)
1689 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1691 "list of Dbf records, with set-like behavior"
1692 _desc = ''
1693 - def __init__(yo, new_records=None, desc=None):
1694 yo._list = []
1695 yo._set = set()
1696 yo._current = -1
1697 if isinstance(new_records, DbfList):
1698 yo._list = new_records._list[:]
1699 yo._set = new_records._set.copy()
1700 yo._current = 0
1701 elif new_records is not None:
1702 for record in new_records:
1703 item = (record.record_table, record.record_number)
1704 if item not in yo._set:
1705 yo._set.add(item)
1706 yo._list.append(item)
1707 yo._current = 0
1708 if desc is not None:
1709 yo._desc = desc
1711 if isinstance(other, DbfList):
1712 result = DbfList()
1713 result._set = yo._set.copy()
1714 result._list[:] = yo._list[:]
1715 for item in other._list:
1716 if item not in result._set:
1717 result._set.add(item)
1718 result._list.append(item)
1719 result._current = 0 if result else -1
1720 return result
1721 return NotImplemented
1723 if isinstance(key, int):
1724 item = yo._list.pop[key]
1725 yo._set.remove(item)
1726 elif isinstance(key, slice):
1727 yo._set.difference_update(yo._list[key])
1728 yo._list.__delitem__(key)
1729 else:
1730 raise TypeError
1732 if isinstance(key, int):
1733 count = len(yo._list)
1734 if not -count <= key < count:
1735 raise IndexError("Record %d is not in list." % key)
1736 return yo._get_record(*yo._list[key])
1737 elif isinstance(key, slice):
1738 result = DbfList()
1739 result._list[:] = yo._list[key]
1740 result._set = set(result._list)
1741 result._current = 0 if result else -1
1742 return result
1743 else:
1744 raise TypeError
1746 return (table.get_record(recno) for table, recno in yo._list)
1748 return len(yo._list)
1754 if yo._desc:
1755 return "DbfList(%s - %d records)" % (yo._desc, len(yo._list))
1756 else:
1757 return "DbfList(%d records)" % len(yo._list)
1759 if isinstance(other, DbfList):
1760 result = DbfList()
1761 result._list[:] = other._list[:]
1762 result._set = other._set.copy()
1763 lost = set()
1764 for item in yo._list:
1765 if item in result._list:
1766 result._set.remove(item)
1767 lost.add(item)
1768 result._list = [item for item in result._list if item not in lost]
1769 result._current = 0 if result else -1
1770 return result
1771 return NotImplemented
1773 if isinstance(other, DbfList):
1774 result = DbfList()
1775 result._list[:] = yo._list[:]
1776 result._set = yo._set.copy()
1777 lost = set()
1778 for item in other._list:
1779 if item in result._set:
1780 result._set.remove(item)
1781 lost.add(item)
1782 result._list = [item for item in result._list if item not in lost]
1783 result._current = 0 if result else -1
1784 return result
1785 return NotImplemented
1787 if item not in yo._set:
1788 yo._set.add(item)
1789 yo._list.append(item)
1791 if table is rec_no is None:
1792 table, rec_no = yo._list[yo._current]
1793 return table.get_record(rec_no)
1799 if yo._list:
1800 yo._current = len(yo._list) - 1
1801 return yo._get_record()
1802 raise DbfError("DbfList is empty")
1804 yo._list = []
1805 yo._set = set()
1806 yo._current = -1
1808 if yo._current < 0:
1809 raise Bof()
1810 elif yo._current == len(yo._list):
1811 raise Eof()
1812 return yo._get_record()
1813 - def extend(yo, new_records):
1814 if isinstance(new_records, DbfList):
1815 for item in new_records._list:
1816 yo._maybe_add(item)
1817 else:
1818 for record in new_records:
1819 yo.append(record)
1820 if yo._current == -1 and yo._list:
1821 yo._current = 0
1822 - def goto(yo, index_number):
1823 if yo._list:
1824 if 0 <= index_number <= len(yo._list):
1825 yo._current = index_number
1826 return yo._get_record()
1827 raise DbfError("index %d not in DbfList of %d records" % (index_number, len(yo._list)))
1828 raise DbfError("DbfList is empty")
1829 - def insert(yo, i, table, record):
1830 item = table, record.record_number
1831 if item not in yo._set:
1832 yo._set.add(item)
1833 yo._list.insert(i, item)
1835 if yo._current < len(yo._list):
1836 yo._current += 1
1837 if yo._current < len(yo._list):
1838 return yo._get_record()
1839 raise Eof()
1840 - def pop(yo, index=None):
1841 if index is None:
1842 table, recno = yo._list.pop()
1843 yo._set.remove((table, recno))
1844 else:
1845 table, recno = yo._list.pop(index)
1846 yo._set.remove((table, recno))
1847 return _get_record(table, recno)
1849 if yo._current >= 0:
1850 yo._current -= 1
1851 if yo._current > -1:
1852 return yo._get_record()
1853 raise Bof()
1861 if yo._list:
1862 yo._current = 0
1863 return yo._get_record()
1864 raise DbfError("DbfList is empty")
1865 - def sort(yo, key=None, reverse=None):
1876 csv.register_dialect('dbf', DbfCsv)
1877
1878 -def _nop(value):
1879 "returns parameter unchanged"
1880 return value
1882 "ensures each tuple is the same length, using filler[-missing] for the gaps"
1883 final = []
1884 for t in tuples:
1885 if len(t) < length:
1886 final.append( tuple([item for item in t] + filler[len(t)-length:]) )
1887 else:
1888 final.append(t)
1889 return tuple(final)
1891 if cp not in code_pages:
1892 for code_page in sorted(code_pages.keys()):
1893 sd, ld = code_pages[code_page]
1894 if cp == sd or cp == ld:
1895 if sd is None:
1896 raise DbfError("Unsupported codepage: %s" % ld)
1897 cp = code_page
1898 break
1899 else:
1900 raise DbfError("Unsupported codepage: %s" % cp)
1901 sd, ld = code_pages[cp]
1902 return cp, sd, ld
1903 -def ascii(new_setting=None):
1910 -def codepage(cp=None):
1911 "get/set default codepage for any new tables"
1912 global default_codepage
1913 cp, sd, ld = _codepage_lookup(cp or default_codepage)
1914 default_codepage = sd
1915 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
1923 version = 'dBase IV w/memos (non-functional)'
1924 _versionabbv = 'db4'
1925 _fieldtypes = {
1926 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
1927 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
1928 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
1929 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
1930 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
1931 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
1932 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
1933 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
1934 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
1935 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1936 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1937 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
1938 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
1939 _memoext = '.dbt'
1940 _memotypes = ('G','M','P')
1941 _memoClass = _VfpMemo
1942 _yesMemoMask = '\x8b'
1943 _noMemoMask = '\x04'
1944 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1945 _variable_fields = ('C','F','N')
1946 _character_fields = ('C','M')
1947 _decimal_fields = ('F','N')
1948 _numeric_fields = ('B','F','I','N','Y')
1949 _supported_tables = ('\x04', '\x8b')
1950 _dbfTableHeader = ['\x00'] * 32
1951 _dbfTableHeader[0] = '\x8b'
1952 _dbfTableHeader[10] = '\x01'
1953 _dbfTableHeader[29] = '\x03'
1954 _dbfTableHeader = ''.join(_dbfTableHeader)
1955 _dbfTableHeaderExtra = ''
1956 _use_deleted = True
1958 "dBase III specific"
1959 if yo._meta.header.version == '\x8b':
1960 try:
1961 yo._meta.memo = yo._memoClass(yo._meta)
1962 except:
1963 yo._meta.dfd.close()
1964 yo._meta.dfd = None
1965 raise
1966 if not yo._meta.ignorememos:
1967 for field in yo._meta.fields:
1968 if yo._meta[field]['type'] in yo._memotypes:
1969 if yo._meta.header.version != '\x8b':
1970 yo._meta.dfd.close()
1971 yo._meta.dfd = None
1972 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
1973 elif not os.path.exists(yo._meta.memoname):
1974 yo._meta.dfd.close()
1975 yo._meta.dfd = None
1976 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1977 break
1978