Jump to content

Sky Slate Blueberry Blackcurrant Watermelon Strawberry Orange Banana Apple Emerald Chocolate
Photo

[AHK_L] ListMFTfiles: NTFS Instant File Search


  • Please log in to reply
96 replies to this topic
wOxxOm
  • Members
  • 371 posts
  • Last active: Feb 20 2015 12:10 PM
  • Joined: 09 Feb 2006
Searches whole drive in just a few seconds or faster even if you have 100000 files or more, and lists file names, optionally matching given regex criteria.

Features:
[*:r35pfbfp]AHK_L Unicode only
[*:r35pfbfp]NTFS drives only
[*:r35pfbfp]No junctions support
[*:r35pfbfp]Searches all files on a drive specified in the first parameter, for example "c:"
[*:r35pfbfp]Optional regex parameter to filter filenames
[*:r35pfbfp]No error reporting, just an empty string is returned
[*:r35pfbfp]Tested on XPsp3, Win7x64
NB. For Vista/Win7 user-level (non-admin) account, add this code at the start:
if not A_IsAdmin
{
	run *runAs "%A_ScriptFullPath%"  ; Requires v1.0.92.01+
	ExitApp
}
Usage example 1 "Get all files":
filelist:=ListMFTfiles("c:")
Usage example 2 "Show in notepad":
filelist:=ListMFTfiles("c:","i)\.(bak|tmp|$$$)$")
file:=fileOpen(A_Temp "\mft.txt","w","utf-8"), file.Write(filelist), file.close()
Run,notepad "%A_Temp%\mft.txt"
Usage example 3 "Simple search GUI":
#NoEnv
#SingleInstance,force
SetBatchLines,-1

if not A_IsAdmin
{
	run *runAs "%A_ScriptFullPath%"  ; Requires v1.0.92.01+
	ExitApp
}

inputbox,s,ListMFTfiles,Folder [regex]`n(no spaces in regex),,300,140,,,,,% substr(A_WinDir,1,2) " i)\.(exe|dll)$"
IfEqual,ErrorLevel,1, ExitApp

t1:=A_TickCount
s:=regExReplace(regExReplace(s,"^\s+",""),"\s+$","")
i:=instr(s," ",true,0)
i:=(i && FileExist(s)="") ? i : strlen(s)+1
filelist:=ListMFTfiles(regExReplace(substr(s,1,i-1),"\s$",""),substr(s,i+1),"|",true,num)
if !filelist
{
	msgbox Nothing found for %s%`nError code: %num%
	exitApp
}

if( strlen(filelist)>1000000 )
{
	StringReplace,filelist,filelist,|,`n,all
	file:=fileOpen(A_Temp "\MFTreader file list.txt","w","utf-8"), file.Write(filelist), file.close()
	Run,notepad "%A_Temp%\MFTreader file list.txt"
	exitapp
}

GuiControl,-Redraw,flChoice
gui,add,listbox,% "R" (num>30 ? 30 : num) " W700 vflChoice gflClick Multi",%filelist%
gui,add,button,default gflRunBtn,&Run
gui,add,button,x+10 gflFolderBtn,&Folder
gui,add,button,x+10 gflSellAllBtn,&Select all
gui,add,button,x+10 gflCopyBtn,&Copy names
gui,add,button,x+10 gflCancel,&Cancel
GuiControl,+Redraw,flChoice
gui,show,autosize,% "ListMFTfiles: " num " files in " (A_TickCount-t1) "ms for " s

HotKey,ifWinActive,% "ahk_id " WinExist("A")
HotKey,Enter,flRunBtn
HotKey,^Enter,flFolderBtn
HotKey,^a,flSellAllBtn
HotKey,^c,flCopyBtn
HotKey,^Ins,flCopyBtn
HotKey,^Insert,flCopyBtn
HotKey,Escape,flCancel
return

#IfWinActive ListMFTfiles ahk_class #32770
flClick:
	IfEqual,A_GuiEvent,DoubleClick, goto flRunBtn
	return
flRunBtn:
	ControlGet,c,Choice,,ListBox1,A
	SplitPath,c,fname,fdir
	run,%c%,%fdir%,UseErrorLevel
	return
flFolderBtn:
	ControlGet,c,Choice,,ListBox1,A
	run,explorer /select`,"%c%",,UseErrorLevel
	return
flSellAllBtn:
	PostMessage,LB_SETSEL:=0x185,1,-1,ListBox1,A
	return
flCopyBtn:
	gui,submit,nohide
	StringReplace,Clipboard,flChoice,|,`n,all
	return
^w::
flCancel:
	ExitApp
#IfWinActive
The code:
; path:
;	where to search, drive or fully specified folder, for example C:\folder
; regex:
;	search for file names that match regex, only file name is matched, default is none
; delim:
;	filelist delimiter, default is newline `n
; showprogress:
;	show minimal progressbar in screen center, default is true
; num:
;	variable that receives number of files returned or error status when trying to obtain:
;	-1,-2: root folder handle or info, -3,-4: path handle or info, -5,-6: volume handle or info,
;	-7: USN journal handle
; plainSearchTimeout: (ms)
;	if greater than 0 (default), plain folder enumeration is used for no more than specified time.
;
; RETURN VALUE:
;	filelist or empty string if error occured (also see 'num' parameter)

ListMFTfiles( path, regex="", delim="`n", showProgress=true, byref num=0, plainSearchTimeout=0 )
{
;=== init
	t0:=A_TickCount
	drive:=substr(path,1,2)
	path:=(substr(path,0)="\") ? path : path "\"

	OPEN_EXISTING:=3
	FILE_FLAG_BACKUP_SEMANTICS:=0x2000000
	SHARE_RW:=3 ;FILE_SHARE_READ | FILE_SHARE_WRITE
	GENERIC_RW:=0xC0000000 ;GENERIC_READ | GENERIC_WRITE

;=== if path can be enumerated within specified time, return filelist at once
	if( plainSearchTimeout>0 )
	{
		num:=0
		bUsePath:=true
		VarSetCapacity(filelist,1000*200) ; 1000 filepaths of ~100 widechars
		loop,%path%*,1,1 ;folders too, with recursion
			if( A_TickCount-t0>plainSearchTimeout )
			{
				bUsePath:=false
				break
			}
			else if( regex="" || regExMatch(A_LoopFileName,regex) )
			{
				filelist.=A_LoopFileLongPath delim
				num++
			}
		if( bUsePath )
		{
			VarSetCapacity(filelist,-1)
			Sort,filelist,D%delim%
			return filelist
		}
	}

;=== get root folder ("\") refnumber

	hRoot:=dllCall("CreateFile","wstr","\\.\" drive "\", "uint",0, "uint",SHARE_RW, "uint",0
					, "uint",OPEN_EXISTING, "uint",FILE_FLAG_BACKUP_SEMANTICS, "uint",0)
	if( hRoot=-1 )
	{
		num:=-1
		return
	}
	;BY_HANDLE_FILE_INFORMATION
	;	0	DWORD dwFileAttributes;
	;	4	FILETIME ftCreationTime;
	;	12	FILETIME ftLastAccessTime;
	;	20	FILETIME ftLastWriteTime;
	;	28	DWORD dwVolumeSerialNumber;
	;	32	DWORD nFileSizeHigh;
	;	36	DWORD nFileSizeLow;
	;	40	DWORD nNumberOfLinks;
	;	44	DWORD nFileIndexHigh;
	;	48	DWORD nFileIndexLow;
	VarSetCapacity(fi,52,0)
	ok:=dllCall("GetFileInformationByHandle", "uint",hRoot, "uint",&fi)
	dllCall("CloseHandle", "uint",hRoot)
	if( ok=0 )
	{
		num:=-2
		return
	}
	dirdict:={}
	ref:=((numget(fi,44)<<32)+numget(fi,48)) ;nFileIndex
	dirdict[ref]:={"name":drive, "parent":"0", "files":{}}

;=== open volume

	hJRoot:=dllCall("CreateFile", "wstr","\\.\" drive, "uint",GENERIC_RW, "uint",SHARE_RW, "uint",0
				, "uint",OPEN_EXISTING, "uint",FILE_FLAG_SEQUENTIAL_SCAN:=0x08000000, "uint",0)
	if( hJRoot=-1 )
	{
		num:=-5
		return
	}

;=== open USN journal

	VarSetCapacity(cujd,16) ;CREATE_USN_JOURNAL_DATA
	numput(0x800000,cujd,0,"uint64")
	numput(0x100000,cujd,8,"uint64")
	ok:=dllCall("DeviceIoControl", "uint",hJRoot, "uint",FSCTL_CREATE_USN_JOURNAL:=0x000900e7
				, "uint",&cujd, "uint",16, "uint",0, "uint",0, "uintp",cb, "uint",0)
	if( ok=0 )
	{
		dllCall("CloseHandle", "uint",hJRoot)
		num:=-6
		return
	}

;=== estimate overall number of files

	;NTFS_VOLUME_DATA_BUFFER
	;	0	LARGE_INTEGER VolumeSerialNumber;
	;	8	LARGE_INTEGER NumberSectors;
	;	16	LARGE_INTEGER TotalClusters;
	;	24	LARGE_INTEGER FreeClusters;
	;	32	LARGE_INTEGER TotalReserved;
	;	40	DWORD         BytesPerSector;
	;	44	DWORD         BytesPerCluster;
	;	48	DWORD         BytesPerFileRecordSegment;
	;	52	DWORD         ClustersPerFileRecordSegment;
	;	56	LARGE_INTEGER MftValidDataLength;
	;	64	LARGE_INTEGER MftStartLcn;
	;	72	LARGE_INTEGER Mft2StartLcn;
	;	80	LARGE_INTEGER MftZoneStart;
	;	88	LARGE_INTEGER MftZoneEnd;
	VarSetCapacity(voldata,96,0)
	mftFiles:=0
	mftFilesMax:=0
	if( dllCall("DeviceIoControl", "uint",hJRoot, "uint",FSCTL_GET_NTFS_VOLUME_DATA:=0x00090064, "uint",0, "uint",0, "uint",&voldata, "uint",96, "uintp",cb, "uint",0) && cb=96 )
		if( i:=numget(voldata,48) )
			mftFilesMax:=numget(voldata,56,"uint64")//i ;MftValidDataLength/BytesPerFileRecordSegment

;=== query USN journal

	;USN_JOURNAL_DATA
	;	0	DWORDLONG UsnJournalID;
	;	8	USN FirstUsn;
	;	16	USN NextUsn;
	;	24	USN LowestValidUsn;
	;	32	USN MaxUsn;
	;	40	DWORDLONG MaximumSize;
	;	48	DWORDLONG AllocationDelta;
	VarSetCapacity(ujd,56,0)
	if( dllCall("DeviceIoControl", "uint",hJRoot, "uint",FSCTL_QUERY_USN_JOURNAL:=0x000900f4, "uint",0, "uint",0, "uint",&ujd, "uint",56, "uintp",cb, "uint",0)=0 )
	{
		dllCall("CloseHandle", "uint",hJRoot)
		num:=-7
		return
	}
	JournalMaxSize:=numget(ujd,40,"uint64")+numget(ujd,48,"uint64") ;MaximumSize+AllocationDelta
	JournalChunkSize:=0x10000 ;1MB chunk, ~10-20 read ops for 150k files
	if( mftFilesMax=0 )
		mftFilesMax:=JournalMaxSize/JournalChunkSize

	t1:=A_TickCount
	if showprogress
		Progress,b p0

;=== enumerate USN journal

	cb:=0
	num:=0
	numD:=0
	VarSetCapacity(pData,8+JournalChunkSize,0)
	dirdict.SetCapacity(JournalMaxSize//(128*50)) ;average file name ~64 widechars, dircount is ~1/50 of filecount

	;MFT_ENUM_DATA
	;	0	DWORDLONG StartFileReferenceNumber;
	;	8	USN LowUsn;
	;	16	USN HighUsn;
	VarSetCapacity(med,24,0)
	numput(numget(ujd,16,"uint64"),med,16,"uint64") ;med.HighUsn=ujd.NextUsn

	while( dllCall("DeviceIoControl", "uint",hJRoot, "uint",FSCTL_ENUM_USN_DATA:=0x000900b3, "uint",&med, "uint",24, "uint",&pData, "uint",8+JournalChunkSize, "uintp",cb, "uint",0) )
	{
		t1a:=A_TickCount
		if showprogress
			Progress,% (mftFiles*80)//mftFilesMax

		pUSN:=&pData+8
		while( cb>8 )
		{
			mftFiles++
			;USN_RECORD
			;	0	DWORD RecordLength;
			;	4	WORD   MajorVersion;
			;	6	WORD   MinorVersion;
			;	8	DWORDLONG FileReferenceNumber;
			;	16	DWORDLONG ParentFileReferenceNumber;
			;	24	USN Usn;
			;	32	LARGE_INTEGER TimeStamp;
			;	40	DWORD Reason;
			;	44	DWORD SourceInfo;
			;	48	DWORD SecurityId;
			;	52	DWORD FileAttributes;
			;	56	WORD   FileNameLength;
			;	58	WORD   FileNameOffset;
			;	60	WCHAR FileName[1];
			fnsize:=numget(pUSN+56,"ushort")
			fname:=strget(pUSN+60,fnsize//2,"UTF-16")
			isdir:=numget(pUSN+52) & 0x10 ;USN.FileAttributes & FILE_ATTRIBUTE_DIRECTORY
			ref:=numget(pUSN+8,"uint64") ;USN.FileReferenceNumber
			refparent:=numget(pUSN+16,"uint64") ;USN.ParentFileReferenceNumber

			if( isdir )
			{
				v:=dirdict[ref]
				if( v="" )
					v:={}, v.files:={}
				v.setCapacity(4) ;4th value 'dir' is created later in resolveFolder()
				v.setCapacity("name",fnsize), v.name:=fname
				v.setCapacity("parent",strlen(refparent)<<1), v.parent:=refparent
				dirdict[ref]:=v
				numD++
			}
			else if( regex="" || regExMatch(fname,regex) )
			{
				v:=dirdict[refparent]
				if( v )
					v:=v.files
				else
					v:={}, dirdict[refparent]:={"files":v}
				v.SetCapacity(ref,fnsize), v[ref]:=fname
				num++
			}

			i:=numget(pUSN+0) ;USN.RecordLength
			pUSN += i
			cb -= i
		}

		nextUSN:=numget(pData,"uint64")
		numput(nextUSN,med,"uint64")

		t1b+=A_TickCount-t1a
	}

	dllCall("CloseHandle", "uint",hJRoot)

	t2:=A_TickCount
	if showprogress
		Progress,80

;=== connect files to parent folders & build new cache
	VarSetCapacity(filelist,num*200) ;average full filepath ~100 widechars
	bPathFilter:=strlen(path)>3 && instr(FileExist(path),"D")>0
	num:=0
	for dk,dv in dirdict
		if( dv.files.getCapacity() )
		{
			dir:=_ListMFTfiles_resolveFolder(dirdict,dk)
			if( !bPathFilter || instr(dir,path)=1 )
				for k,v in dv.files
					filelist.=dir v delim, num++
		}

	dirdict=
	VarSetCapacity(filelist,-1)

	t3:=A_TickCount
	if showprogress
		Progress,85

;=== sort
	Sort,filelist,D%delim%

	t4:=A_TickCount
	if showprogress
		Progress,OFF

	;msgbox % "init`tenum`twinapi`tconnect`tsort`ttotal, ms`n" (t1-t0) "`t" t1b "`t" (t2-t1-t1b) "`t" (t3-t2) "`t" (t4-t3) "`t" (t4-t0) "`n`ndirs:`t" numD "`nfiles:`t" num
	return filelist
}

_ListMFTfiles_resolveFolder( byref dirdict, byref ddref )
{
	p:=dirdict[ddref], pd:=p.dir
	if( !pd )
	{
		pd:=(p.parent ? _ListMFTfiles_resolveFolder(dirdict,p.parent) : "") p.name "\"
		p.setCapacity("dir",strlen(pd)*2)
		p.dir:=pd
	}
	return pd
}
History:
06/01: reduce memory consumption by ~25%, overall speed-up by ~25%, new params: path & plainSearchTimeout, percentage in progressbar is more accurate, enhanced example 3.
05/30: reduce memory consumption by 2.5 times
04/18: speed-up folder resolving stage 10x

  • Guests
  • Last active:
  • Joined: --
I Posted Image you.

skrommel
  • Members
  • 193 posts
  • Last active: Jun 07 2010 08:30 AM
  • Joined: 30 Jul 2004
:O Wow! Next step: Text search, file information, file watch?

Also, TrayTip is slow, use a timer, or something like

oldtime:=time
time:=A_Now
If (time<>oldtime)
traytip,,% "Reading USN " round(A_index*JournalChunkSize/JournalMaxSize*100,1) "%..."

Skrommel

wOxxOm
  • Members
  • 371 posts
  • Last active: Feb 20 2015 12:10 PM
  • Joined: 09 Feb 2006
I've added 3rd example, a simple GUI.

Also, TrayTip is slow, use a timer, or something like

I've changed to an optional Progress window, but actually the update is called just a few times due to a large JournalChunkSize.

S0und
  • Members
  • 100 posts
  • Last active: Apr 08 2015 10:07 AM
  • Joined: 16 Feb 2007
Everything search engine
Locate files and folders by name instantly.

wOxxOm
  • Members
  • 371 posts
  • Last active: Feb 20 2015 12:10 PM
  • Joined: 09 Feb 2006
S0und, this is irrelevant here, it was already posted in utilities subforum.

Learning one
  • Members
  • 1483 posts
  • Last active: Jan 02 2016 02:30 PM
  • Joined: 04 Apr 2009
This is really rocket fast wOxxOm! Thank you! :)

Tested and works fine on Windows 7 Home Premium 64-bit

My Website • Recommended: AutoHotkey Unicode 32-bit • Join DropBox, Copy


capitalH
  • Members
  • 64 posts
  • Last active: Apr 05 2012 05:18 AM
  • Joined: 23 Apr 2010
I made some small changes (sharing is caring)
1) added a sort option
2) added a subfolder option (since the regex only applies to the filename, and not the path. For example to search for text files in My documents
filelist:=ListMFTfiles("c:","i)\.(txt)$","`r`n",A_MyDocuments)

ListMFTfiles( drive, regex="", delim="`n", subfolder="", showprogress=true, sort=false, byref num=0 )
{
;=== get root folder ("\") refnumber
   SHARE_RW:=3 ;FILE_SHARE_READ | FILE_SHARE_WRITE
   hRoot:=dllCall("CreateFileW",wstr,"\\.\" drive "\", uint,0, uint,SHARE_RW, uint,0, uint,OPEN_EXISTING:=3, uint,FILE_FLAG_BACKUP_SEMANTICS:=0x2000000, uint,0)
   ifEqual,hRoot,-1, return
   ;BY_HANDLE_FILE_INFORMATION
   ;   0   DWORD dwFileAttributes;
   ;   4   FILETIME ftCreationTime;
   ;   12   FILETIME ftLastAccessTime;
   ;   20   FILETIME ftLastWriteTime;
   ;   28   DWORD dwVolumeSerialNumber;
   ;   32   DWORD nFileSizeHigh;
   ;   36   DWORD nFileSizeLow;
   ;   40   DWORD nNumberOfLinks;
   ;   44   DWORD nFileIndexHigh;
   ;   48   DWORD nFileIndexLow;
   VarSetCapacity(fi,52,0)
   ok:=dllCall("GetFileInformationByHandle", uint,hRoot, uint,&fi), dllCall("CloseHandle", uint,hRoot)
   ifEqual,ok,0, return
   dirdict:={}
   rootDirKey:="" ((numget(fi,44)<<32)+numget(fi,48))
   dirdict[rootDirKey]:={"name":drive, "parent":"0"}

;=== open USN journal
   GENERIC_RW:=0xC0000000 ;GENERIC_READ | GENERIC_WRITE
   hJRoot:=dllCall("CreateFileW", wstr,"\\.\" drive, uint,GENERIC_RW, uint,SHARE_RW, uint,0, uint,OPEN_EXISTING:=3, uint,0, uint,0)
   ifEqual,hJRoot,-1, return
   cb:=0
   VarSetCapacity(cujd,16) ;CREATE_USN_JOURNAL_DATA
   numput(0x800000,cujd,0,"uint64")
   numput(0x100000,cujd,8,"uint64")
   if( dllCall("DeviceIoControl", uint,hJRoot, uint,FSCTL_CREATE_USN_JOURNAL:=0x000900e7, uint,&cujd, uint,16, uint,0, uint,0, uintp,cb, uint,0)=0 )
   {
      dllCall("CloseHandle", uint,hJRoot)
      return
   }

;=== prepare data to query USN journal
   ;USN_JOURNAL_DATA
   ;   0   DWORDLONG UsnJournalID;
   ;   8   USN FirstUsn;
   ;   16   USN NextUsn;
   ;   24   USN LowestValidUsn;
   ;   32   USN MaxUsn;
   ;   40   DWORDLONG MaximumSize;
   ;   48   DWORDLONG AllocationDelta;
   VarSetCapacity(ujd,56,0)
   if( dllCall("DeviceIoControl", uint,hJRoot, uint,FSCTL_QUERY_USN_JOURNAL:=0x000900f4, uint,0, uint,0, uint,&ujd, uint,56, uintp,cb, uint,0)=0 )
   {
      dllCall("CloseHandle", uint,hJRoot)
      return
   }
   JournalMaxSize:=numget(ujd,40,"uint64")

;=== enumerate USN journal
   cb:=0
   filedict:={}
   filedict.SetCapacity(JournalMaxSize//(128*10))
   dirdict.SetCapacity(JournalMaxSize//(128*10))
   JournalChunkSize:=0x100000
   VarSetCapacity(pData,8+JournalChunkSize,0)
   ;MFT_ENUM_DATA
   ;   0   DWORDLONG StartFileReferenceNumber;
   ;   8   USN LowUsn;
   ;   16   USN HighUsn;
   VarSetCapacity(med,24,0)
   numput(numget(ujd,16,"uint64"),med,16,"uint64") ;med.HighUsn=ujd.NextUsn

   if showprogress
      Progress,b p0
   while dllCall("DeviceIoControl", uint,hJRoot, uint,FSCTL_ENUM_USN_DATA:=0x000900b3, uint,&med, uint,24, uint,&pData, uint,8+JournalChunkSize, uintp,cb, uint,0)
   {
      pUSN:=&pData+8
      ;USN_RECORD
      ;   0   DWORD RecordLength;
      ;   4   WORD   MajorVersion;
      ;   6   WORD   MinorVersion;
      ;   8   DWORDLONG FileReferenceNumber;
      ;   16   DWORDLONG ParentFileReferenceNumber;
      ;   24   USN Usn;
      ;   32   LARGE_INTEGER TimeStamp;
      ;   40   DWORD Reason;
      ;   44   DWORD SourceInfo;
      ;   48   DWORD SecurityId;
      ;   52   DWORD FileAttributes;
      ;   56   WORD   FileNameLength;
      ;   58   WORD   FileNameOffset;
      ;   60   WCHAR FileName[1];
      while cb>60
      {
         ref:="" numget(pUSN+8,"uint64") ;USN.FileReferenceNumber
         refparent:="" numget(pUSN+16,"uint64") ;USN.ParentFileReferenceNumber
         fn:=strget(pUSN+60,numget(pUSN+56,"ushort")//2,"UTF-16") ;USN.FileName
         if( numget(pUSN+52) & 0x10 ) ;USN.FileAttributes & FILE_ATTRIBUTE_DIRECTORY
            dirdict[ref]:={"name":fn, "parent":refparent}
         else
            if( regex="" || regExMatch(fn,regex) )
               filedict[ref]:={"name":fn, "parent":refparent}
         i:=numget(pUSN+0) ;USN.RecordLength
         pUSN += i
         cb -= i
      }
      numput(numget(pData,"uint64"),med,"uint64")
      if showprogress
         Progress,% round(A_index*JournalChunkSize/JournalMaxSize*100)
   }
   dllCall("CloseHandle", uint,hJRoot)

;=== connect files to parent folders
   VarSetCapacity(filelist,filedict.getCapacity()*128)
   for k,v in filedict
   {
      v2:=v, fn:=v.name
	  ;;traytip %fn%, 
      while v2.parent
         p:=dirdict[v2.parent], fn:=p.name "\" fn, v2:=p
	if instr(fn, subfolder)=1	
	{
      filelist.=fn delim
      num++
	 } 
   }
   if showprogress
      Progress,99
   VarSetCapacity(filelist,-1)
   if sort
	Sort,filelist,D%delim%
   if showprogress
      Progress,OFF
   return filelist
}

Edit: and by the way it works great on Windows XP, except that the progress indicator is messed up. JournalMaxSize gets reported as 8388608, it goes through 47 loops of JournalChunkSize=1048576, and progress goes up to 587%.

wOxxOm
  • Members
  • 371 posts
  • Last active: Feb 20 2015 12:10 PM
  • Joined: 09 Feb 2006
capitalH, this is interesting, you motivated me to do what I was reluctant to do initially so I made folder resolving 10 times faster, feel free to merge it into your mod - or allow me to incorporate your options into my script. Regarding journal size, I don't know yet how to fix it, probably adding UJD.AllocationDelta to JournalMaxSize should fix it, see the code.

  • Guests
  • Last active:
  • Joined: --
Thanks wOxxOm and capitalH
Amazingly fast :shock:
This should be Ahk_l default
I just don't understand the subfolder option in capitalH's post
Can you please document it better or make an example?

jballi
  • Members
  • 1029 posts
  • Last active:
  • Joined: 01 Oct 2005
Cool beans. 8) Thanks for sharing.

Maestr0
  • Members
  • 652 posts
  • Last active: Aug 17 2019 06:07 PM
  • Joined: 18 Oct 2008
Brilliant!
Although not quite as fast as everything.exe, still very very cool!

Is there a way to allow for more than one drive to be mentioned?
Maybe by adding a function like:
ListMFTfiles0( drive, regex="", delim="`n", showprogress=true, byref num=0 )
{
	if drive not contains |	; meaning: only one drive
		filelist2:=ListMFTfiles(drive,search)
	else	; more than one drive, so parse and get list
	{
		loop, parse, Drive, |
		{
			if A_Index = 1
				filelist2:=ListMFTfiles(A_LoopField,search)
			else
				filelist2:=filelist2 . "`n" . ListMFTfiles(A_LoopField,search)
		}
	}
	return filelist2
}


Maestr0
  • Members
  • 652 posts
  • Last active: Aug 17 2019 06:07 PM
  • Joined: 18 Oct 2008
For some reason unbeknownst to me example 3 (but really all the examples) does not (always) work, and I just get the report that there are no .exe or .lnk files on C: ... why would this be?
"Nothing found for c: i)\.(exe|lnk)$"

I hit OK and I get the message immediately. When it does work (which is rarely, sadly) I see a progress bar. I see no progress bar when it doesn't work.

What am I doing wrong?

wOxxOm
  • Members
  • 371 posts
  • Last active: Feb 20 2015 12:10 PM
  • Joined: 09 Feb 2006
If you use AHK_L unicode build as stated in requirements then the only way to know what's happening I can think of, is debugging the script on your pc.

Maestr0
  • Members
  • 652 posts
  • Last active: Aug 17 2019 06:07 PM
  • Joined: 18 Oct 2008

If you use AHK_L unicode build as stated in requirements then the only way to know what's happening I can think of, is debugging the script on your pc.

Using AHK_L Unicode, I'd checked that.
How's this as a debugging version?
ListMFTfiles( drive, regex="", delim="`n", showprogress=true, byref num=0, debugging="1" )
{
	if debugging
		outputdebug % "drive: " drive . "`nregex: " . regex . "`ndelim: " . delim . "`nshowprogress: " . showprogress . "`nnum: " . num
;=== get root folder ("\") refnumber
   SHARE_RW:=3 ;FILE_SHARE_READ | FILE_SHARE_WRITE
   hRoot:=dllCall("CreateFileW",wstr,"\\.\" drive "\", uint,0, uint,SHARE_RW, uint,0, uint,OPEN_EXISTING:=3, uint,FILE_FLAG_BACKUP_SEMANTICS:=0x2000000, uint,0)
	if debugging
		outputdebug % "hroot: " hroot
   ifEqual,hRoot,-1, return
   ;BY_HANDLE_FILE_INFORMATION
   ;   0   DWORD dwFileAttributes;
   ;   4   FILETIME ftCreationTime;
   ;   12   FILETIME ftLastAccessTime;
   ;   20   FILETIME ftLastWriteTime;
   ;   28   DWORD dwVolumeSerialNumber;
   ;   32   DWORD nFileSizeHigh;
   ;   36   DWORD nFileSizeLow;
   ;   40   DWORD nNumberOfLinks;
   ;   44   DWORD nFileIndexHigh;
   ;   48   DWORD nFileIndexLow;
   VarSetCapacity(fi,52,0)
   ok:=dllCall("GetFileInformationByHandle", uint,hRoot, uint,&fi), dllCall("CloseHandle", uint,hRoot)
  	if debugging
		outputdebug % "ok: " ok
   ifEqual,ok,0, return
   dirdict:={}
   rootDirKey:="" ((numget(fi,44)<<32)+numget(fi,48))
   dirdict[rootDirKey]:={"name":drive, "parent":"0"}
  	if debugging
		outputdebug % "rootdirkey: " rootdirkey

;=== open USN journal
   GENERIC_RW:=0xC0000000 ;GENERIC_READ | GENERIC_WRITE
   hJRoot:=dllCall("CreateFileW", wstr,"\\.\" drive, uint,GENERIC_RW, uint,SHARE_RW, uint,0, uint,OPEN_EXISTING:=3, uint,0, uint,0)
  	if debugging
		outputdebug % "hJRoot: " hJRoot
   [color=#FF0000]ifEqual,hJRoot,-1, return[/color]
   cb:=0
   VarSetCapacity(cujd,16) ;CREATE_USN_JOURNAL_DATA
   numput(0x800000,cujd,0,"uint64")
   numput(0x100000,cujd,8,"uint64")
   if( dllCall("DeviceIoControl", uint,hJRoot, uint,FSCTL_CREATE_USN_JOURNAL:=0x000900e7, uint,&cujd, uint,16, uint,0, uint,0, uintp,cb, uint,0)=0 )
   {
      dllCall("CloseHandle", uint,hJRoot)
		if debugging
			outputdebug % "USN Journal not created"
      return
   }
   else
   {
  		if debugging
			outputdebug % "USN Journal created"
	}


;=== prepare data to query USN journal
   ;USN_JOURNAL_DATA
   ;   0   DWORDLONG UsnJournalID;
   ;   8   USN FirstUsn;
   ;   16   USN NextUsn;
   ;   24   USN LowestValidUsn;
   ;   32   USN MaxUsn;
   ;   40   DWORDLONG MaximumSize;
   ;   48   DWORDLONG AllocationDelta;
   VarSetCapacity(ujd,56,0)
   if( dllCall("DeviceIoControl", uint,hJRoot, uint,FSCTL_QUERY_USN_JOURNAL:=0x000900f4, uint,0, uint,0, uint,&ujd, uint,56, uintp,cb, uint,0)=0 )
   {
		if debugging
			outputdebug % "Problem preparing data to query USN journal, closing Handle"
	   dllCall("CloseHandle", uint,hJRoot)
      return
   }
   JournalMaxSize:=numget(ujd,40,"uint64")+numget(ujd,48,"uint64") ;USN_JOURNAL_DATA.MaximumSize + USN_JOURNAL_DATA.AllocationDelta
  	if debugging
		outputdebug % "JournalMaxSize: " JournalMaxSize

;=== enumerate USN journal
   cb:=0
   filedict:={}
   filedict.SetCapacity(JournalMaxSize//(128*10))
   dirdict.SetCapacity(JournalMaxSize//(128*10))
   JournalChunkSize:=0x100000
   VarSetCapacity(pData,8+JournalChunkSize,0)
   ;MFT_ENUM_DATA
   ;   0   DWORDLONG StartFileReferenceNumber;
   ;   8   USN LowUsn;
   ;   16   USN HighUsn;
   VarSetCapacity(med,24,0)
   numput(numget(ujd,16,"uint64"),med,16,"uint64") ;med.HighUsn=ujd.NextUsn

   if showprogress
      Progress,b p0
   while dllCall("DeviceIoControl", uint,hJRoot, uint,FSCTL_ENUM_USN_DATA:=0x000900b3, uint,&med, uint,24, uint,&pData, uint,8+JournalChunkSize, uintp,cb, uint,0)
   {
      pUSN:=&pData+8
      ;USN_RECORD
      ;   0   DWORD RecordLength;
      ;   4   WORD   MajorVersion;
      ;   6   WORD   MinorVersion;
      ;   8   DWORDLONG FileReferenceNumber;
      ;   16   DWORDLONG ParentFileReferenceNumber;
      ;   24   USN Usn;
      ;   32   LARGE_INTEGER TimeStamp;
      ;   40   DWORD Reason;
      ;   44   DWORD SourceInfo;
      ;   48   DWORD SecurityId;
      ;   52   DWORD FileAttributes;
      ;   56   WORD   FileNameLength;
      ;   58   WORD   FileNameOffset;
      ;   60   WCHAR FileName[1];
      while cb>60
      {
         fn:=strget(pUSN+60,numget(pUSN+56,"ushort")//2,"UTF-16") ;USN.FileName
         if( numget(pUSN+52) & 0x10 ) ;USN.FileAttributes & FILE_ATTRIBUTE_DIRECTORY
         {
            ref:="" numget(pUSN+8,"uint64") ;USN.FileReferenceNumber
            refparent:="" numget(pUSN+16,"uint64") ;USN.ParentFileReferenceNumber
            dirdict[ref]:={"name":fn, "parent":refparent}
         }
         else
            if( regex="" || regExMatch(fn,regex) )
            {
               ref:="" numget(pUSN+8,"uint64") ;USN.FileReferenceNumber
               refparent:="" numget(pUSN+16,"uint64") ;USN.ParentFileReferenceNumber
               filedict[ref]:={"name":fn, "parent":refparent}
            }
         i:=numget(pUSN+0) ;USN.RecordLength
         pUSN += i
         cb -= i
      }
      numput(numget(pData,"uint64"),med,"uint64")
      if showprogress
         Progress,% round(A_index*JournalChunkSize/JournalMaxSize*90)
   }
  	if debugging
		outputdebug % "i: " i
   dllCall("CloseHandle", uint,hJRoot)

;=== connect files to parent folders
   if showprogress
      Progress,90
   VarSetCapacity(filelist,filedict.getCapacity()*128)
   num:=0
   for k,v in filedict
   {
      filelist.=resolveFolder(dirdict,v.parent) v.name delim
      num++
   }
   if showprogress
      Progress,95
   VarSetCapacity(filelist,-1)
  	if debugging
		outputdebug % "filelist generated, " . i . " hits"
   Sort,filelist,D%delim%
   if showprogress
      Progress,OFF
   return filelist
}

resolveFolder( dirdict,ddref )
{
   p:=dirdict[ddref], pd:=p.dir
   return pd ? pd : (p.dir:=(p.parent ? resolveFolder(dirdict,p.parent) : "") p.name "\")
}

The script stops at "ifEqual,hJRoot,-1, return"