mkexfat.py

import utils, struct, disk, os, sys, math
import pprint

from exFAT import *

# Note: expanded and compressed tables generated by this functions may differ
# from MS's FORMAT (different locales?), but Windows and CHKDSK accept them!

# Experimenting with wrong compressed Up-Case tables showed that in many cases
# CHKDSK accepts them and signals no error, but Windows puts the filesystem
# in Read-Only mode instead!
def gen_upcase():
    "Generate the full, expanded (128K) UpCase table"
    tab = []
    for i in range(65536):
        i = ord(unichr(i).upper())
        tab += [struct.pack('<H', i)]
    return bytearray().join(tab)


def gen_upcase_compressed():
    "Generate a compressed (about 4K) UpCase table"
    tab = []
    run = -1
    for i in xrange(65536):
        u = unichr(i)
        U = u.upper()
        if u != U:
            rl = i-run
            if run > -1 and rl > 2:
                # Replace chars with range
                del tab[len(tab)-rl:]
                tab += [unichr(0xFFFF), unichr(rl)]
            run = -1
        else:
            if run < 0: run = i
        tab += [U]
    return bytearray(u''.join(tab).encode('utf-16le'))



nodos_asm_78h = '\xB8\xC0\x07\x8E\xD8\xBE\x93\x00\xAC\x08\xC0\x74\x0A\xB4\x0E\xBB\x07\x00\xCD\x10\xE9\xF1\xFF\xF4\xE9\xFC\xFF\x4E\x4F\x20\x44\x4F\x53\x00'


def calc_cluster(size):
    "Returns a cluster adequate to volume size, MS FORMAT style (exFAT)"
    c = 9 # min cluster: 512 (2^9)
    v = 26 # min volume: 64 MiB (2^26)
    for i in range(17):
        if size <= 2**v: return 2**c
        c+=1
        v+=1
        if v == 29: v+=4
        if v == 39: v+=1
    return (2<<25) # Maximum cluster: 32 MiB


#~ #####
#~ The layout of an exFAT file system is far more complex than old FAT.
#
#~ At start we have a Volume Boot Record of 12 sectors made of:
#~ - a boot sector area of 9 sectors, where the first one contains the usual
#~   FS descriptors and boot code. However, the boot code can span sectors;
#~ - an OEM parameter sector, which must be zeroed if unused;
#~ - a reserved sector (MS FORMAT does not even blank it!);
#~ - a checksum sector, filled with the same DWORD containing the calculated
#~   checksum of the previous 11 sectors.
#
#~ A backup copy of these 12 sectors must follow immediately.
#
#~ Then the FAT region with a single FAT (except in the -actually unsupported-
#~ T-exFAT). It hasn't to be consecutive to the previous region; however, it can't
#~ legally lay inside the clusters heap (like NTFS $MFT) nor after it.
#
#~ Finally, the Data region (again, it can reside far from FAT area) where the
#~ root directory is located.
#
#~ But the root directory must contain (and is normally preceeded by):
#~ - a special Bitmap file, where allocated clusters are set;
#~ - a special Up-Case file (compressed or uncompressed) for Unicode file name
#~   comparisons.
#~ Those are "special" since marked with single slots of special types (0x81, 0x82)
#~ instead of standard file/directory slots group (0x85, 0xC0, 0xC1).
#
#~ FAT is set and valid only for fragmented files. However, it must be always set for
#~ Root, Bitmap and Up-Case, even if contiguous.
#####
def exfat_mkfs(stream, size, sector=512, params={}):
    "Make an exFAT File System on stream. Returns 0 for success."

    sectors = size/sector

    if 'reserved_size' in params:
        reserved_size = params['reserved_size']*sector
        if reserved_size < 24*sector:
            reserved_size = 24*sector
    else:
        # At least 24 sectors required for Boot region & its backup
        #~ reserved_size = 24*sector
        reserved_size = 65536 # FORMAT default

    if 'fat_copies' in params:
        fat_copies = params['fat_copies']
    else:
        fat_copies = 1 # default: best setting

    if 'dataregion_padding' in params:
        dataregion_padding = params['dataregion_padding']
    else:
        dataregion_padding = 0 # additional space between FAT region and Data region

    allowed = {} # {cluster_size : fsinfo}

    for i in range(9, 25): # cluster sizes 0.5K...32M
        fsinfo = {}
        cluster_size = (2**i)
        clusters = (size - reserved_size) / cluster_size
        # cluster_size increase? FORMAT seems to reserve more space than minimum
        fat_size = (4*(clusters+2)+sector-1)/sector * sector
        # round it to cluster_size, or memory page size or something?
        fat_size = (fat_size+cluster_size-1)/cluster_size * cluster_size
        required_size = cluster_size*clusters + fat_copies*fat_size + reserved_size + dataregion_padding
        while required_size > size:
            clusters -= 1
            fat_size = (4*(clusters+2)+sector-1)/sector * sector
            fat_size = (fat_size+cluster_size-1)/cluster_size * cluster_size
            required_size = cluster_size*clusters + fat_copies*fat_size + reserved_size + dataregion_padding
        if clusters < 1 or clusters > 0xFFFFFFFF:
            continue
        fsinfo['required_size'] = required_size # space occupied by FS
        fsinfo['reserved_size'] = reserved_size # space reserved before FAT#1
        fsinfo['cluster_size'] = cluster_size
        fsinfo['clusters'] = clusters
        fsinfo['fat_size'] = fat_size # space occupied by a FAT copy
        allowed[cluster_size] = fsinfo

    if not allowed:
        if clusters < 1:
            print "Can't apply exFAT with less than 1 cluster!"
            return -1
        else:
            print "Too many clusters to apply exFAT: aborting."
            return -1

    #~ print "* MKFS exFAT INFO: allowed combinations for cluster size:"
    #~ pprint.pprint(allowed)

    fsinfo = None

    if 'wanted_cluster' in params:
        if params['wanted_cluster'] in allowed:
            fsinfo = allowed[params['wanted_cluster']]
        else:
            print "Specified cluster size of %d is not allowed for exFAT: aborting..." % params['wanted_cluster']
            return -1
    else:
        fsinfo = allowed[calc_cluster(size)]

    boot = boot_exfat()
    boot.chJumpInstruction = '\xEB\x76\x90' # JMP opcode is mandatory, or CHKDSK won't recognize filesystem!
    boot._buf[0x78:0x78+len(nodos_asm_78h)] = nodos_asm_78h # insert assembled boot code
    boot.chOemID = '%-8s' % 'EXFAT'
    boot.u64PartOffset = 0x3F
    boot.u64VolumeLength = sectors
    # We can put FAT far away from reserved area, if we want...
    boot.dwFATOffset = (reserved_size+sector-1)/sector
    boot.dwFATLength = (fsinfo['fat_size']+sector-1)/sector
    # Again, we can put clusters heap far away from usual
    boot.dwDataRegionOffset = boot.dwFATOffset + boot.dwFATLength + dataregion_padding
    boot.dwDataRegionLength = fsinfo['clusters']
    # We'll calculate this after writing Bitmap and Up-Case
    boot.dwRootCluster = 0
    boot.dwVolumeSerial = exFATDirentry.GetDosDateTimeEx()[0]
    boot.wFSRevision = 0x100
    boot.wFlags = 0
    boot.uchBytesPerSector = int(math.log(sector)/math.log(2))
    boot.uchSectorsPerCluster = int(math.log(fsinfo['cluster_size']/sector) / math.log(2))
    boot.uchFATCopies = fat_copies
    boot.uchDriveSelect = 0x80
    boot.wBootSignature = 0xAA55

    boot.__init2__()

    # Blank the FAT area
    stream.seek(boot.fatoffs)
    blank = bytearray(sector)
    for i in xrange(boot.dwFATLength):
        stream.write(blank)

    # Initialize the FAT
    clus_0_2 = '\xF8\xFF\xFF\xFF\xFF\xFF\xFF\xFF'
    stream.seek(boot.fatoffs)
    stream.write(clus_0_2)

    # Make a Bitmap slot
    b = bytearray(32); b[0] = 0x81
    bitmap = exFATDirentry(b, 0)
    bitmap.dwStartCluster = 2 # default, but not mandatory
    bitmap.u64DataLength = (boot.dwDataRegionLength+7)/8

    # Blank the Bitmap Area
    stream.seek(boot.cl2offset(bitmap.dwStartCluster))
    for i in range((bitmap.u64DataLength+boot.cluster-1)/boot.cluster):
        stream.write(bytearray(boot.cluster))

    # Make the Up-Case table and its file slot (following the Bitmap)
    start = bitmap.dwStartCluster + (bitmap.u64DataLength+boot.cluster-1)/boot.cluster

    # Write the compressed Up-Case table
    stream.seek(boot.cl2offset(start))
    table = gen_upcase_compressed()
    stream.write(table)

    # Make the Up-Case table slot
    b = bytearray(32); b[0] = 0x82
    upcase = exFATDirentry(b, 0)
    upcase.dwChecksum = boot.GetChecksum(table, True)
    upcase.dwStartCluster = start
    upcase.u64DataLength = len(table)

    # Finally we can fix the root cluster!
    boot.dwRootCluster = upcase.dwStartCluster + (upcase.u64DataLength+boot.cluster-1)/boot.cluster

    # Write the VBR area (first 12 sectors) and its backup
    stream.seek(0)
    # Write boot & VBR sectors
    stream.write(boot.pack())
    # Since we haven't large boot code, all these are empty
    empty = bytearray(512); empty[-2] = 0x55; empty[-1] = 0xAA
    for i in range(8):
        stream.write(empty)
    # OEM parameter sector must be totally blank if unused (=no 0xAA55 signature)
    stream.write(bytearray(512))
    # This sector is reserved, can have any content
    stream.write(bytearray(512))

    # Read the first 11 sectors and get their 32-bit checksum
    stream.seek(0)
    vbr = stream.read(sector*11)
    checksum = struct.pack('<I', boot.GetChecksum(vbr))

    # Fill the checksum sector
    checksum = sector/4 * checksum

    # Write it, then the backup of the 12 sectors
    stream.write(checksum)
    stream.write(vbr)
    stream.write(checksum)

    # Blank the root directory cluster
    stream.seek(boot.root())
    stream.write(bytearray(boot.cluster))

    # Initialize root Dirtable
    boot.stream = stream
    fat = FAT(stream, boot.fatoffs, boot.clusters(), bitsize=32, exfat=True)

    # Mark the FAT chain for Bitmap, Up-Case and Root
    fat.mark_run(bitmap.dwStartCluster, (bitmap.u64DataLength+boot.cluster-1)/boot.cluster)
    fat.mark_run(upcase.dwStartCluster, (upcase.u64DataLength+boot.cluster-1)/boot.cluster)
    fat[boot.dwRootCluster] = fat.last

    # Initialize the Bitmap and mark the allocated clusters so far
    bmp = Bitmap(boot, fat, bitmap.dwStartCluster)
    bmp.set(bitmap.dwStartCluster, (bitmap.u64DataLength+boot.cluster-1)/boot.cluster)
    bmp.set(upcase.dwStartCluster, (upcase.u64DataLength+boot.cluster-1)/boot.cluster)
    bmp.set(boot.dwRootCluster)

    boot.bitmap = bmp
    root = Dirtable(boot, fat, boot.dwRootCluster)

    # Write an empty Volume Label (optional), the Bitmap and UpCase slots (mandatory)
    b = bytearray(32); b[0] = 0x3
    label = exFATDirentry(b, 0)
    root.stream.write(label.pack())
    root.stream.write(bitmap.pack())
    root.stream.write(upcase.pack())
    
    root.flush() # commit all changes to disk immediately, or volume won't be usable!

    sizes = {0:'B', 10:'KiB',20:'MiB',30:'GiB',40:'TiB',50:'EiB'}
    k = 0
    for k in sorted(sizes):
        if (fsinfo['required_size'] / (1<<k)) < 1024: break

    free_clusters = boot.dwDataRegionLength - (bitmap.u64DataLength+boot.cluster-1)/boot.cluster - (upcase.u64DataLength+boot.cluster-1)/boot.cluster - 1
    print "Successfully applied exFAT to a %.02f %s volume.\n%d clusters of %.1f KB.\n%.02f %s free in %d clusters." % (fsinfo['required_size']/float(1<<k), sizes[k], fsinfo['clusters'], fsinfo['cluster_size']/1024.0, free_clusters*boot.cluster/float(1<<k), sizes[k], free_clusters)
    print "\nFAT Region @0x%X, Data Region @0x%X, Root (cluster #%d) @0x%X" % (boot.fatoffs, boot.cl2offset(2), boot.dwRootCluster, boot.cl2offset(boot.dwRootCluster))

    return 0



if __name__ == '__main__':
    import Volume
    import logging
    logging.basicConfig(level=logging.DEBUG, filename='mkexfat.log', filemode='w')

    if len(sys.argv) < 2:
        print "mkexfat error: you must specify a target volume to apply an exFAT file system!"
        sys.exit(1)

    dsk = Volume.vopen(sys.argv[1], 'r+b', 'disk')

    params={}
    if len(sys.argv)==3:
        params['wanted_cluster'] = int(sys.argv[2])

    exfat_mkfs(dsk, dsk.size, params=params)