How process DOS file

[mikechy]

Hi all,

I should process many really big ASCII files. The first line in all files it is necessary to pass. The length of all records in file can be various and the information concerning length of following record is possible to read through in the first line of each record. Is it possible to use in such case DOS driver? Thanks.

Mike

 

[ShankarJ]

Hi Mike,

Data Section ::

AsciiFile FILE,DRIVER('DOS','/FILEBUFFERS=1024 /QUICKSCAN=ON'),PRE(ASC)
Record RECORD,PRE()
Data STRING(4096) ! chunks of data that you want to read on each GET()
END
END

RecBuffer STRING(1024) ! max size of records (longest line + 2)

Code Section :


AsciiFile{PROP:Name} = '...'

CLOSE(AsciiFile)
OPEN(AsciiFile, 40h)
IF ERRORCODE()
MESSAGE(CLIP(ERRORFILE()) & ' : ' & ERROR() & ' [' & ERRORCODE() & ']|' & 'File Error : ' & FILEERROR() & ' [' & FILEERRORCODE() & ']','E R R O R - OPEN()')
EXIT
END

FileSize# = BYTES(AsciiFile) ; FilePtr# = 1 ; RecPtr# = 1

LOOP
IF (FilePtr# + 4096) > FileSize#
BytesToRead# = FileSize# - FilePtr# + 1
ELSE
BytesToRead# = 4096
END

GET(AsciiFile, FilePtr#, BytesToRead#)
IF ERRORCODE() = 35
BREAK
ELSIF ERRORCODE()
MESSAGE(CLIP(ERRORFILE()) & ' : ' & ERROR() & ' [' & ERRORCODE() & ']|' & 'File Error : ' & FILEERROR() & ' [' & FILEERRORCODE() & ']','E R R O R - GET()')
BREAK
END

StartPos# = 1 ; SepPos# = 0
LOOP
SepPos# = INSTRING('<13,10>', ASCata, 1, StartPos#)
IF SepPos#
RecBuffer[RecPtr# : (RecPtr# + SepPos# - StartPos# - 1)] = ASCata[StartPos# : (SepPos# - 1)]

RecPtr# = RecPtr# + SepPos# - StartPos# - 1

... do your processing ...

RecPtr# = 1 ; CLEAR(RecBuffer)

StartPos# = SepPos# + 2
ELSE
CLEAR(RecBuffer)

RecBuffer = ASCata[StartPos# : BytesToRead#]

RecPtr# = BytesToRead# - StartPos# + 2

BREAK
END
END

FilePtr# += BytesToRead#

IF FilePtr# > FileSize#
BREAK
END
END

CLOSE(AsciiFile)

Regards

 

[mikechy]

Hi ShankarJ,
Thanks for your reply. I think that your suggesting should work for me.
Season greetings.
Regards,
Mike

 

[mikechy]

Hi ShankarJ,

Happy New Year!

I can process my files, reading it as DOS file. But I have a small problem when I checked performance of the simple code:

FileSize# = BYTES(dosFile) ; FilePtr# = 1
Chunk# = 4096

time0# = clock()
LOOP
j# += 1
IF (FilePtr# + Chunk#) > FileSize#
BytesToRead# = FileSize# - FilePtr# + 1
ELSE
BytesToRead# = Chunk#
END

GET(dosFile, FilePtr#, BytesToRead#)
IF ERRORCODE()
MESSAGE(ERROR())
BREAK
END

StartPos# = 1
FilePtr# += BytesToRead#

IF FilePtr# > FileSize#
BREAK
END
END
message(j# & '|' & CLOCK() - time0#)
CLOSE(dosFile)
CLEAR(DOS:Record)

When I run such code, and I open the same file then every time I have little bit different time of performance. Any thought on it?

Thanks,
Mike

 

[ShankarJ]

Hi Mike,

Happy New Year to you as well.

If you have Disk Caching, the first time will be slow and the second/third time will be faster. Is that what is happening?

If you want to try the API approach, below is a sample program to calculate the CRC checksum of a file. Maybe you can adapt it to your needs.

                             PROGRAM

                             MAP
                               MODULE('CLARION')
                                 sj_CRC32(*STRING Buffer, ULONG Bytes, ULONG Start), ULONG, RAW, NAME('CLA$CRC32')
                               END

                               MODULE('WINAPI')
                                 sj_CreateFile(*CSTRING,ULONG,ULONG,LONG,ULONG,ULONG,UNSIGNED=0),UNSIGNED,RAW,PASCAL,NAME('CreateFileA')
                                 sj_GetFileSize(UNSIGNED,*ULONG),ULONG,PASCAL,NAME('GetFileSize')
                                 sj_ReadFile(UNSIGNED,LONG,ULONG,*ULONG,LONG),BOOL,PASCAL,RAW,NAME('ReadFile')
                                 sj_CloseHandle(UNSIGNED),BOOL,PASCAL,PROC,NAME('CloseHandle')
                               END

                               CalcFileCRC(STRING inpFileName),ULONG
                             END

GLO:FileName                 STRING(255)
GLO:CRCValue                 ULONG

Window WINDOW('Calculate CRC Value of File'),AT(,,254,36),FONT('Arial',8,,FONT:bold),CENTER,WALLPAPER('BROWN.JPG'), |
         TILED,SYSTEM,GRAY
       PROMPT('File to calculate CRC'),AT(3,5),USE(?GLO:FileName:Prompt),TRN
       ENTRY(@s255),AT(68,5,171,10),USE(GLO:FileName),LEFT,REQ
       BUTTON,AT(241,5,10,10),USE(?GLO:FileName:Lookup),ICON(ICON:Ellipsis)
       PROMPT('CRC Value'),AT(3,20),USE(?GLO:CRCValue:Prompt),TRN
       ENTRY(@N_18B),AT(68,20,89,10),USE(GLO:CRCValue),SKIP,RIGHT(1),READONLY
       BUTTON('&Calculate CRC'),AT(161,19,90,12),USE(?CalcCRC)
     END

  CODE

  OPEN(Window)
  ACCEPT
     CASE EVENT()
     OF EVENT:OpenWindow
       DISPLAY()

     OF EVENT:CloseWindow
       BREAK

     OF EVENT:Accepted
       CASE FIELD()
       OF ?GLO:FileName:Lookup
         IF FILEDIALOG('Choose File ...', GLO:FileName, , FILE:LongName)
            DISPLAY(?GLO:FileName)
         END

       OF ?CalcCRC
         GLO:CRCValue = CalcFileCRC(GLO:FileName)

         DISPLAY()
       END
     END
  END
  CLOSE(Window)

  RETURN


CalcFileCRC                  PROCEDURE(STRING inpFileName)!,ULONG

RetVal                       LONG

szFileName                   CSTRING(256)
hFile                        LONG
lFileSize                    ULONG
lBytesRead                   ULONG
sBuffer                      &STRING
lCRCValue                    ULONG

INVALID_HANDLE_VALUE         EQUATE(-1)
GENERIC_READ                 EQUATE(80000000h)
OPEN_EXISTING                EQUATE(3)

  CODE

  lCRCValue = 0

  szFileName = CLIP(inpFileName)

  hFile = sj_CreateFile(szFileName, GENERIC_READ, 0, 0, OPEN_EXISTING, 0, 0)

  IF hFile <> INVALID_HANDLE_VALUE
     lFileSize = sj_GetFileSize(hFile, lBytesRead)

     IF lFileSize
        sBuffer &= NEW (STRING(lFileSize))

        RetVal = sj_ReadFile(hFile, ADDRESS(sBuffer), lFileSize, lBytesRead, 0)

        IF RetVal
           lCRCValue = sj_CRC32(sBuffer, SIZE(sBuffer), 0)
        ELSE
           MESSAGE('ERROR Reading File')
        END

        DISPOSE(sBuffer)
     ELSE
        MESSAGE('ZERO File Size')
     END

     sj_CloseHandle(hFile)
  ELSE
     MESSAGE('ERROR Opening File')
  END

  RETURN lCRCValue

Regards

 

[mikechy]

Hi ShankarJ,
Yeah, it's exactly that I had: first time is about in 10 times slower than the subsequent.
I'll try your suggesting and will inform on result.
Many thanks.
Regards,
Mike