Great! Much abliged...
Is there any way to speed up the scanning of large files more? I found two (fast) command line utilities that beat this script considerably in speed:
rehash.exe (has other checksum options too)
crc32.exe (fastest of the crc32.exe's I found)
Time to scan a 700 MB iso file:
- script: 30 seconds (buffer-size: 32768, latest AutoHotkey beta)
- rehash: 19 seconds
- crc32: 18 seconds
Other command line programs I tried (but were slower than the ones mentioned above):
-
crc32.exe
-
crc32.exe
-
crc.exe
testscript for AHK code
Code:
#NoEnv
SetBatchLines -1
;Critical On
;Process, Priority, , Realtime
StartTime := A_TickCount
aBuffer := 1024 * 32
VarSetCapacity(data,aBuffer) ; allocate 4KB buffer (change to your taste)
file := "E:\Downloads\gos-3.1-gadgets-20081205.iso" ;A_ScriptFullPath ; put your filename here
FileGetSize Sz, %file%
c := 0, offs := -aBuffer
Loop % Sz//aBuffer { ; for each block of data
BinRead(file, data, aBuffer, offs+=aBuffer)
c := CRC32(data,aBuffer,~c) ; compute accumulated CRC
}
If (m:=mod(Sz,aBuffer)) { ; the slack
BinRead(file, data, m, offs+=aBuffer)
c := CRC32(data,m,~c)
}
; c = CRC here
SetFormat Integer, Hex
crc := c+0 ; show CRC32 in hex
SetFormat Integer, D
ElapsedTime := Round((A_TickCount - StartTime)/1000)
MsgBox % "It took " ElapsedTime " seconds to scan`n`n" file "`n`nBuffer size: " aBuffer " kB`nCRC: " crc
Return
BinRead(file, ByRef data, n=0, offset=0) {
h := DllCall("CreateFile",Str,file, UInt,0x80000000, UInt,3, UInt,0, UInt,3, UInt,0, UInt,0)
DllCall("SetFilePointerEx",UInt,h, Int64,offset, UIntP,U, Int,2*(offset<0))
m := DllCall("GetFileSize",UInt,h, Int64P,r)
If n not between 1 and %m%
n := m
VarSetCapacity(data, n)
DllCall("ReadFile",UInt,h, Str,data, UInt,n, UIntP,r, UInt,0)
DllCall("CloseHandle", UInt,h)
Return r
}
CRC32(ByRef Buffer, Bytes=0, Start=-1) {
Static CRC32, CRC32LookupTable
If (CRC32 = "") {
MCode(CRC32,"33c06a088bc85af6c101740ad1e981f12083b8edeb02d1e94a75ec8b542404890c82403d0001000072d8c3")
VarSetCapacity(CRC32LookupTable, 1024)
DllCall(&CRC32, "uint",&CRC32LookupTable, "cdecl")
MCode(CRC32,"558bec33c039450c7627568b4d080fb60c08334d108b55108b751481e1ff000000c1ea0833148e403b450c89551072db5e8b4510f7d05dc3")
}
If Bytes <= 0
Bytes := StrLen(Buffer)
Return DllCall(&CRC32, "uint",&Buffer, "uint",Bytes, "int",Start, "uint",&CRC32LookupTable, "cdecl uint")
}
MCode(ByRef code, hex) { ; allocate memory and write Machine Code there
VarSetCapacity(code,StrLen(hex)//2)
Loop % StrLen(hex)//2
NumPut("0x" . SubStr(hex,2*A_Index-1,2), code, A_Index-1, "Char")
}
testscript for console programs:
Code:
#NoEnv
SetBatchLines -1
;Critical On
;Process, Priority, , Realtime
StartTime := A_TickCount
;aBuffer := 1024 * 4
fileToScan := "E:\Downloads\gos-3.1-gadgets-20081205.iso"
CMDdir := A_ScriptDir "\bin"
CMDin := A_ScriptDir "\bin\rehash.exe -none -crc32 -norcrsv -f """ fileToScan """"
crc := CMDret_RunReturn(CMDin, CMDdir)
ElapsedTime := Round((A_TickCount - StartTime)/1000)
MsgBox % "It took " ElapsedTime " seconds to scan`n`n" fileToScan "`n`nBuffer size: " aBuffer " kB`nCRC: " crc ; show CRC32 in hex
Return
; ******************************************************************
; CMDret-AHK functions
; version 1.10 beta
;
; Updated: Dec 5, 2006
; by: corrupt
; Code modifications and/or contributions made by:
; Laszlo, shimanov, toralf, Wdb
; ******************************************************************
; Usage:
; CMDin - command to execute
; WorkingDir - full path to working directory (Optional)
; ******************************************************************
; Known Issues:
; - If using dir be sure to specify a path (example: cmd /c dir c:\)
; or specify a working directory
; - Running 16 bit console applications may not produce output. Use
; a 32 bit application to start the 16 bit process to receive output
; ******************************************************************
; Additional requirements:
; - none
; ******************************************************************
; Code Start
; ******************************************************************
CMDret_RunReturn(CMDin, WorkingDir=0)
{
Global cmdretPID
tcWrk := WorkingDir=0 ? "Int" : "Str"
idltm := A_TickCount + 20
CMsize = 1
VarSetCapacity(CMDout, 1, 32)
VarSetCapacity(sui,68, 0)
VarSetCapacity(pi, 16, 0)
VarSetCapacity(pa, 12, 0)
Loop, 4 {
DllCall("RtlFillMemory", UInt,&pa+A_Index-1, UInt,1, UChar,12 >> 8*A_Index-8)
DllCall("RtlFillMemory", UInt,&pa+8+A_Index-1, UInt,1, UChar,1 >> 8*A_Index-8)
}
IF (DllCall("CreatePipe", "UInt*",hRead, "UInt*",hWrite, "UInt",&pa, "Int",0) <> 0) {
Loop, 4
DllCall("RtlFillMemory", UInt,&sui+A_Index-1, UInt,1, UChar,68 >> 8*A_Index-8)
DllCall("GetStartupInfo", "UInt", &sui)
Loop, 4 {
DllCall("RtlFillMemory", UInt,&sui+44+A_Index-1, UInt,1, UChar,257 >> 8*A_Index-8)
DllCall("RtlFillMemory", UInt,&sui+60+A_Index-1, UInt,1, UChar,hWrite >> 8*A_Index-8)
DllCall("RtlFillMemory", UInt,&sui+64+A_Index-1, UInt,1, UChar,hWrite >> 8*A_Index-8)
DllCall("RtlFillMemory", UInt,&sui+48+A_Index-1, UInt,1, UChar,0 >> 8*A_Index-8)
}
IF (DllCall("CreateProcess", Int,0, Str,CMDin, Int,0, Int,0, Int,1, "UInt",0, Int,0, tcWrk, WorkingDir, UInt,&sui, UInt,&pi) <> 0) {
Loop, 4
cmdretPID += *(&pi+8+A_Index-1) << 8*A_Index-8
Loop {
idltm2 := A_TickCount - idltm
If (idltm2 < 10) {
DllCall("Sleep", Int, 10)
Continue
}
IF (DllCall("PeekNamedPipe", "uint", hRead, "uint", 0, "uint", 0, "uint", 0, "uint*", bSize, "uint", 0 ) <> 0 ) {
Process, Exist, %cmdretPID%
IF (ErrorLevel OR bSize > 0) {
IF (bSize > 0) {
VarSetCapacity(lpBuffer, bSize+1)
IF (DllCall("ReadFile", "UInt",hRead, "Str", lpBuffer, "Int",bSize, "UInt*",bRead, "Int",0) > 0) {
IF (bRead > 0) {
TRead += bRead
VarSetCapacity(CMcpy, (bRead+CMsize+1), 0)
CMcpy = a
DllCall("RtlMoveMemory", "UInt", &CMcpy, "UInt", &CMDout, "Int", CMsize)
DllCall("RtlMoveMemory", "UInt", &CMcpy+CMsize, "UInt", &lpBuffer, "Int", bRead)
CMsize += bRead
VarSetCapacity(CMDout, (CMsize + 1), 0)
CMDout=a
DllCall("RtlMoveMemory", "UInt", &CMDout, "UInt", &CMcpy, "Int", CMsize)
VarSetCapacity(CMDout, -1) ; fix required by change in autohotkey v1.0.44.14
}
}
}
}
ELSE
break
}
ELSE
break
idltm := A_TickCount
}
cmdretPID=
DllCall("CloseHandle", UInt, hWrite)
DllCall("CloseHandle", UInt, hRead)
}
}
IF (StrLen(CMDout) < TRead) {
VarSetCapacity(CMcpy, TRead, 32)
TRead2 = %TRead%
Loop {
DllCall("RtlZeroMemory", "UInt", &CMcpy, Int, TRead)
NULLptr := StrLen(CMDout)
cpsize := Tread - NULLptr
DllCall("RtlMoveMemory", "UInt", &CMcpy, "UInt", (&CMDout + NULLptr + 2), "Int", (cpsize - 1))
DllCall("RtlZeroMemory", "UInt", (&CMDout + NULLptr), Int, cpsize)
DllCall("RtlMoveMemory", "UInt", (&CMDout + NULLptr), "UInt", &CMcpy, "Int", cpsize)
TRead2 --
IF (StrLen(CMDout) > TRead2)
break
}
}
StringTrimLeft, CMDout, CMDout, 1
Return, CMDout
}