Fixed and enhanced

- exFAT code is now dramatically faster in mapping free space (particularly when volume is almost empty)
- test tools updated (new fat_mkfs code), added VHDX test
- fixed a bug in new FAT(12,16) formatter

FATtools/exFAT.py

@@ -95,14 +95,14 @@ def root(self):
         return self.cl2offset(self.dwRootCluster)
 
     def checkvbr(self):
+        "Calculates and compares VBR checksum to test VBR integrity"
         if not self.stream: return 0
         sector = 1 << self.uchBytesPerSector
         self.stream.seek(0)
         s = self.stream.read(sector*11)
         calc_crc = self.GetChecksum(s)
-        s = self.stream.read(sector)
+        s = self.stream.read(sector) # checksum sector
         stored_crc = struct.unpack('<I',s[:4])[0]
-        print(calc_crc, stored_crc)
         if calc_crc != stored_crc:
             raise exFATException("FATAL: exFAT Volume Boot Region is corrupted, bad checksum!")
 
@@ -117,7 +117,6 @@ def GetChecksum(s, UpCase=False):
         return hash
 
 
-
 def upcase_expand(s):
     "Expands a compressed Up-Case table"
     i = 0
@@ -192,11 +191,20 @@ def map_free_space(self):
                 first_free = -1
                 run_length = -1
                 while j < len(s)*8:
-                    if not j%8 and s[j//8] == 0xFF:
-                        if run_length > 0: break
-                        j+=8
-                        continue
-                    if s[j//8] & (1 << (j%8)):
+                    # Most common case should be all-0|1
+                    if not j%8: # if byte start
+                        if not s[j//8]: # if empty byte
+                            if first_free < 0:
+                                first_free = j+2+i*8
+                                run_length = 0
+                            run_length += 8
+                            j+=8
+                            continue
+                        if s[j//8]==0xFF: # if full byte
+                            if run_length > 0: break
+                            j+=8
+                            continue
+                    if s[j//8] & (1 << (j%8)): # test middle bit
                         if run_length > 0: break
                         j+=1
                         continue

FATtools/mkfat.py

@@ -280,7 +280,10 @@ def fat_mkfs(stream, size, sector=512, params={}):
 
     # Erase FAT(s) area
     stream.seek(boot.fat())
-    to_blank = boot.uchFATCopies * (boot.wSectorsPerFAT | boot.dwSectorsPerFAT) * boot.wBytesPerSector
+    if fat_bits == 32:
+        to_blank = boot.uchFATCopies * boot.dwSectorsPerFAT * boot.wBytesPerSector
+    else:
+        to_blank = boot.uchFATCopies * boot.wSectorsPerFAT * boot.wBytesPerSector
     blank = bytearray(2<<20)
     while to_blank: # 6x faster than sectored technique on large FAT!
         n = min(2<<20, to_blank)

FATtools/utils.py

@@ -1,5 +1,6 @@
 # -*- coding: cp1252 -*-
 import io, struct, os
+from FATtools.debug import log
 DEBUG=int(os.getenv('FATTOOLS_DEBUG', '0'))
 
 class myfile(io.FileIO):

FATtools/vhdutils.py

@@ -590,7 +590,7 @@ def write1(self, s):
                 if s[i:i+put] == self.zero[:put]:
                     i+=put
                     self._pos+=put
-                    if DEBUG&16: log("block #%d @0x%X is zeroed, virtualizing write", self._pos//self.block, (block*self.block)+self.header.dwBlocksOffset)
+                    if DEBUG&16: log("block #%d @0x%X is zeroed, virtualizing write", self._pos//self.block, (block*self.block)+self.header.u64TableOffset)
                     continue
                 # allocates a new block at end before writing
                 self.stream.seek(-512, 2) # overwrites old footer

samples/mk_fdos12_boot_floppy.py

@@ -35,7 +35,7 @@
 
 # Format & add boot code
 f = Volume.vopen(sys.argv[1], 'r+b', 'disk')
-mkfat.fat12_mkfs(f, floppy_size)
+mkfat.fat_mkfs(f, floppy_size)
 f.seek(0)
 boot = f.read(512)
 bootcode = base64.b64decode(bootable['boot'])

samples/stress.py

@@ -95,7 +95,9 @@ def write(self):
 def stress(opts, args):
     "Randomly populates and erases a tree of random files and directories (for test purposes)"
     root = Volume.vopen(args[0], 'r+b') # auto-opens first useful filesystem
-
+    if type(root) == str:
+        print('Could not vopen "%s", returned %s' % (args[0],root))
+        sys.exit(1)
     dirs_made, files_created, files_erased = 0,0,0
 
     tree = GenRandTree()

samples/test_tools.py

@@ -38,7 +38,7 @@ def test(img_file, fssize=32<<20, fat_type='exfat'):
 
     if len(sys.argv)>2 and sys.argv[2]=='mbr':
         print("Creating a MBR partition on disk")
-        gpt = partutils.partition(f, 'mbr', mbr_type=6)
+        gpt = partutils.partition(f, 'mbr')
     else:
         print("Creating a GPT partition on disk")
         gpt = partutils.partition(f)
@@ -51,12 +51,8 @@ def test(img_file, fssize=32<<20, fat_type='exfat'):
     log('Opened %s', f)
     if fat_type == 'exfat':
         fmt = mkfat.exfat_mkfs
-    elif fat_type == 'fat32':
-        fmt = mkfat.fat32_mkfs
-    elif fat_type == 'fat16':
-        fmt = mkfat.fat16_mkfs
-    elif fat_type == 'fat12':
-        fmt = mkfat.fat12_mkfs
+    else:
+        fmt = mkfat.fat_mkfs
     if len(sys.argv)>2 and sys.argv[2]=='mbr':
         fmt(f, f.size)
     else:
@@ -110,13 +106,13 @@ class Opts():
     opts = Opts()
     opts.threshold=60
     opts.file_size=1<<20
-    opts.programs=63
+    opts.programs=63 # bits mask to select tests to run
     #~ opts.programs=31 # exclude buggy dir cleaning
     opts.debug=7
     opts.sha1=1
-    opts.sha1chk=0
+    opts.sha1chk=0 # set to check generated checksums
     opts.fix=0
-    #~ stress.seed(4)
+    #~ stress.seed(2) # set to repeat a fixed pattern
     if VHD_MODE:
         stress.stress(opts, [img_file[:-4]+'_delta.vhd'])
     else:

samples/test_tools_vdi.py

@@ -51,12 +51,8 @@ def test(img_file, fssize=32<<20, fat_type='exfat'):
     log('Opened %s', f)
     if fat_type == 'exfat':
         fmt = mkfat.exfat_mkfs
-    elif fat_type == 'fat32':
-        fmt = mkfat.fat32_mkfs
-    elif fat_type == 'fat16':
-        fmt = mkfat.fat16_mkfs
-    elif fat_type == 'fat12':
-        fmt = mkfat.fat12_mkfs
+    else:
+        fmt = mkfat.fat_mkfs
     if len(sys.argv)>2 and sys.argv[2]=='mbr':
         fmt(f, f.size)
     else:

samples/test_tools_vhdx.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+import os, sys, glob, ctypes, uuid, shutil, logging
+import hexdump
+
+DEBUG=int(os.getenv('FATTOOLS_DEBUG', '0'))
+
+logging.basicConfig(level=logging.DEBUG, filename='test_tools_vhdx.log', filemode='w')
+
+from FATtools.debug import log
+from FATtools import Volume, mkfat, vhdxutils, partutils
+import stress
+
+
+def printn(s): print(s)
+
+def test(img_file, fssize=32<<20, fat_type='exfat'):
+    log("Creating a blank %.02f MiB Dynamic VHDX disk image", (fssize/(1<<20)))
+    print("Creating a blank %.02f MiB Dynamic VHDX disk image" % (fssize/(1<<20)))
+    vhdxutils.mk_dynamic(img_file, fssize, upto=40<<30, overwrite='yes')
+
+    f = Volume.vopen(img_file, 'r+b', 'disk')
+    if f == 'EINV':
+        print('Invalid disk or image file specified to test!')
+        sys.exit(1)
+
+    if len(sys.argv)>2 and sys.argv[2]=='mbr':
+        print("Creating a MBR partition on disk")
+        gpt = partutils.partition(f, 'mbr')
+    else:
+        print("Creating a GPT partition on disk")
+        gpt = partutils.partition(f)
+    f.close() # always close, to avoid tstamp problems!
+
+    print("Applying FAT File System on partition:", fat_type)
+    log("Applying FAT File System on partition: %s", fat_type)
+    f = Volume.vopen(img_file, 'r+b', 'partition0')
+    print('Opened', f)
+    log('Opened %s', f)
+    if fat_type == 'exfat':
+        fmt = mkfat.exfat_mkfs
+    else:
+        fmt = mkfat.fat_mkfs
+    if len(sys.argv)>2 and sys.argv[2]=='mbr':
+        fmt(f, f.size)
+    else:
+        fmt(f, (gpt.partitions[0].u64EndingLBA-gpt.partitions[0].u64StartingLBA+1)*512)
+    f.close()
+
+    #~ root = openpart(DISK, 'r+b').open()
+    #~ root.create('a.txt').write('CIAO')
+
+    print("Injecting a tree")
+    log("Injecting a tree")
+
+    def mktree():
+        try:
+            os.mkdir('t')
+            os.mkdir('t/a')
+            os.mkdir('t/a/a1')
+            os.mkdir('t/a/a1/a2')
+        except WindowsError:
+            pass
+        for base in ('t/a/a1/a2', 't/a/a1', 't/a'):
+            for i in range(20):
+                pn = base+'/File%02d.txt'%i
+                open(pn,'w').write(pn)
+    mktree()
+    root = Volume.vopen(img_file, 'r+b')
+    subdir = root.mkdir('T')
+    Volume.copy_tree_in('.\T', subdir, printn, 2)
+    root.flush()
+    #~ root.close() # always close, to avoid tstamp problems!
+
+    print("Creating a blank %.02f MiB Differencing VHDX disk image, linked to previous one" % (fssize/(1<<20)))
+    vhdxutils.mk_diff(img_file[:-5]+'_delta.vhdx', img_file, overwrite='yes')
+
+    root = Volume.vopen(img_file[:-5]+'_delta.vhdx', 'r+b')
+    root.create('a.txt').write(b'CIAO')
+    root.rmtree('T')
+    root.flush()
+
+    subdir = root.mkdir('T')
+    Volume.copy_tree_in('.\T', subdir, printn, 2)
+    root.flush()
+
+    shutil.rmtree('t')
+
+    print("Running stress test...")
+    class Opts():
+     pass
+
+    opts = Opts()
+    opts.threshold=60
+    opts.file_size=1<<20
+    opts.programs=63 # bits mask to select tests to run
+    #~ opts.programs=31 # exclude buggy dir cleaning
+    opts.debug=7
+    opts.sha1=1
+    opts.sha1chk=0 # set to check generated checksums
+    opts.fix=0
+    #~ stress.seed(2) # set to repeat a fixed pattern
+    stress.stress(opts, [img_file[:-5]+'_delta.vhdx'])
+
+
+
+if __name__ == '__main__':
+    fmts = ['fat12', 'fat16', 'fat32', 'exfat']
+    for fmt in fmts:
+        test(sys.argv[1], fat_type=fmt)

samples/test_tools_vmdk.py

@@ -52,12 +52,8 @@ def test(img_file, fssize=64<<20, fat_type='exfat'):
     log('Opened %s', f)
     if fat_type == 'exfat':
         fmt = mkfat.exfat_mkfs
-    elif fat_type == 'fat32':
-        fmt = mkfat.fat32_mkfs
-    elif fat_type == 'fat16':
-        fmt = mkfat.fat16_mkfs
-    elif fat_type == 'fat12':
-        fmt = mkfat.fat12_mkfs
+    else:
+        fmt = mkfat.fat_mkfs
     if len(sys.argv)>2 and sys.argv[2]=='mbr':
         fmt(f, f.size)
     else: