Mem-Map File demo using API accessing up to 18 exabytes

[Evenbit]

A 'quick hack' but shows the basic approach.  Allocation Granularity is typically 64 KB.

Code Select Expand


//  FileSay.hla
//  converted from a C demo from
//  a Jeffrey Richter book
//  Shows how to mem-map files up to 18 EB, which is
//  1 quintillion, or 1,152,921,504,606,846,976 bytes

program FileSay;
#include( "w.hhf" )
#include( "stdlib.hhf" )

type
    bignum:
        union
            tot     :qword;
            record
                lo      :dword;
                hi      :dword;
            endrecord;
        endunion;
       
static

    hFile           :dword;
    hFileMapping    :dword;
    pbFile          :dword;
    mysi            :w.SYSTEM_INFO;
    qwFileSize      :bignum;
    qwFileOffset    :bignum;
    dwBytesInBlock  :dword;

begin FileSay;

stdout.puts(nl nl);
mov(0, qwFileOffset.lo); // make sure offsets are zero
mov(0, qwFileOffset.hi);
arg.c();
if (eax = 2) then

    w.GetSystemInfo(mysi); // fill SYSTEM_INFO structure

    arg.v(1);
    w.CreateFile(eax,
                 w.GENERIC_READ,
                 w.FILE_SHARE_READ,
                 0,
                 w.OPEN_EXISTING,
                 w.FILE_FLAG_SEQUENTIAL_SCAN,
                 0);

    mov(eax, hFile);
    if (eax = w.INVALID_HANDLE_VALUE) then
        w.ExitThread(0); // just another way to exit
    endif;

    w.CreateFileMapping(hFile, 0, w.PAGE_READONLY, 0, 0, 0);

    mov(eax, hFileMapping);
    if (eax = 0) then
        w.CloseHandle(hFile);
        w.ExitThread(0);
    endif;

    lea(ebx, qwFileSize.hi);
    w.GetFileSize(hFile, [ebx]);
    mov(eax, qwFileSize.lo);
    // 'GetFileSize' shouldn't give us any problems,
    // but we will do error-checking for good messure
    if (eax = $FFFFFFFF) then
        w.GetLastError();
        if (eax != w.NO_ERROR) then
            w.CloseHandle(hFileMapping);
            w.CloseHandle(hFile);
            w.ExitThread(0);
        endif;
    endif;

    forever
        // now we pick the 'chunck' size
        // usually about 64 KB or less
        mov(qwFileSize.hi, eax);
        if (eax = 0) then
            mov(qwFileSize.lo, eax);
            mov(mysi.dwAllocationGranularity, ebx);
            if (eax < ebx) then
                mov(eax, dwBytesInBlock);
            else;
                mov(ebx, dwBytesInBlock);
            endif;
        else;
            mov(mysi.dwAllocationGranularity, ebx);
            mov(ebx, dwBytesInBlock);
        endif;

        w.MapViewOfFile(hFileMapping,
                         w.FILE_MAP_READ,
                         qwFileOffset.hi,
                         qwFileOffset.lo,
                         dwBytesInBlock);
        mov(eax, pbFile);

//  \/ Start of file operations \/

        mov(dwBytesInBlock, edx);
        mov(0, ecx);
       
        myloop:
        mov([eax+ecx], bl);
        stdout.putc(bl);
        inc(ecx);
        cmp(ecx, edx);
        jne myloop;

//  /\ End of file operations /\
       
        w.UnmapViewOfFile(pbFile);

        // Increment Offset quadword
        mov(qwFileOffset.lo, ecx);
        mov(qwFileOffset.hi, edx);
        mov(dwBytesInBlock, eax);
        mov(0, ebx);
        add(eax, ecx);
        adc(ebx, edx);
        mov(ecx, qwFileOffset.lo);
        mov(edx, qwFileOffset.hi);

        // Decrement Size quadword
        mov(qwFileSize.lo, ecx);
        mov(qwFileSize.hi, edx);
        mov(dwBytesInBlock, eax);
        mov(0, ebx);
        sub(eax, ecx);
        sbb(ebx, edx);
        mov(ecx, qwFileSize.lo);
        mov(edx, qwFileSize.hi);
        if (edx = 0) then
            if (ecx = 0) then
                break;
            endif;
        endif;
    endfor;

    w.CloseHandle(hFileMapping);
    w.CloseHandle(hFile);
    stdout.newln();

else;

    stdout.puts("  FILESAY  -  will echo a textfile to STDOUT" nl);
    stdout.puts("              handles files up to 18 ExaBytes" nl nl);
    stdout.puts("    USAGE>   FileSay Textfile.txt" nl);
   
endif;

end FileSay;


[V Coder]

My program works when I allocate memory for a heap data structure with malloc. To use a memory mapped file instead, so I have used the following code. The program creates the file upon execution, but it crashes immediately on trying to read or write (zero) the data.

Please help. Thanks

Code Select Expand

static
hDFile: dword;
hDFileMap: dword;
mem_seed: int32;
readonly
DataFile: string := "LSS_DATA.lss";

//Program
   // Create memory mapped file for the seed checkpoints

w.CreateFile ( DataFile,
w.GENERIC_READ | w.GENERIC_WRITE,
0,
NULL,
w.OPEN_ALWAYS,
w.FILE_ATTRIBUTE_TEMPORARY | w.FILE_FLAG_RANDOM_ACCESS,
NULL);
mov (eax, hDFile);

w.CreateFileMapping ( hDFile,
NULL,
w.PAGE_READWRITE,
0, // 4GB file size
60000,
NULL);
mov (eax, hDFileMap);

w.MapViewOfFile (hDFileMap,
w.FILE_MAP_WRITE, //w.FILE_MAP_ALL_ACCESS,
0,
0,
0);

mov (eax,mem_seed); // seeds

        fileio.put(rptH, "File mapping: ", mem_seed, nl);

xor (edx,edx); xor (ecx,ecx); pxor (mm0, mm0);
   clmem2:
movq ([eax+ecx], mm0); add (8, ecx); jne clmem2;
// movq (mm0, [eax+ecx]); add (8, ecx); jne clmem2;
// mov (edx, [eax+ecx]); sub (4, ecx); jne clmem2;