Function for clicking on delete-button

[Almost_there]

Here I have snipped out a working function (original script is at about 7k lines) that basically search for delete-button, and then click at the position where the button was found.
This is made for a program with a very non-consistent user interface, there is several graphical "versions" of the same button and it might be located at different places at toolbar depending on what element user has selected. The user interface requires the user to almost exclusively use the mouse for any action, eg. the keyboard delete button doesn't work - user have to use the mouse and locate the delete button.

Another problem with the user interface that the function deals with - the buttons on the program interface require that mouse hovers over for a while, before it can accept mouse click.

Also - the functions that deals with image search does store the last found position of image. At next call to same image search function, it only use imagesearch in the area that has same size as the image.

There is also a function that display the indexes and the orders and hit ratio for each index, so the user can easily see what function finds it's image most times.

Also there is some thing I've found out myself to improve the script:

• orderOfFunctionsSorted and countingEachIndex don't need to exist globally. I did make those global this because I originally planned a onExit() function that would write the order to an INI-file.
• need to catch errors from KlikkHoverVentFargeChange() (function is used by other functions in main script - 7k lines) that will stop the function in case delete-button in the program user interface doesn't react on mouse hovering over the button within a given time.
• some functions need to be renamed (easier to understand it purposes), and also the image names that doesn't completely match the names of functions.

Pointed out at in thread where it was originally posted - I added my own question to this comments:

• nnnik: Script is said to be "cancerous" -- may nnnik explain in detail.
• serg : "Gdip_ImageSearch" by MasterFocus is much faster than native "ImageSearch" -- Does it work on x64? Can Gdip_ImageSearch specify color variations?
• nnnik: Maybe a Shape search algorythm -- Doesn't that require reading a lot of "PixelGetColor" to fill a pixel table? Have my doubt in that.

This is as far as I have got in my quest to make the auto-find-and-click-delete-button be as fast as possible. Still, it might be a noticeable delay, especially when running a program on remote desktop.
Nb: Large part of the delay is because the program require mouse pointer to hover over the button, I cannot change that.

Appreciate any feedback that contributes to even better image search performance. But - it must work at win64 and support color variations.
I attach some sample images that show the image used by imagesearch (no because forum bug).

Code: Select all

#NoEnv  ; Recommended for performance and compatibility with future AutoHotkey releases.
SendMode Input  ; Recommended for new scripts due to its superior speed and reliability.
SetWorkingDir %A_ScriptDir%  ; Ensures a consistent starting directory.

orderOfFunctionsSorted := [6,5,4,3,1,2,7,8]  ; May read this from ini-file but for this specific user interface, that would not gain any advantages
countingEachIndex := [0,0,0,0,0,0,0,0]

; Search for delete-button. Maximum performance and use as little resources as possible.
sletteknappFinnTrykk() {
	Global orderOfFunctionsSorted, countingEachIndex
	
	mainLoop_sletteknapp:
	Loop, 8
	{
		func_index := orderOfFunctionsSorted[A_Index]
		returverdi := sletteknappFinnTrykk_U%func_index%()
		If (returverdi == 1) {
			If (A_Index == 1) {
				countingEachIndex[1]++
				Break, mainLoop_sletteknapp
			}
			Else {
				; ACTION 1 - add "points" to whatever index that lead to first function that returned 1
				countingEachIndex[A_Index]++
				index_last_used_last := A_Index -1
				
				; ACTION 2 - sorting
				loop_search_all_decending_sletteknapp:
				Loop, %index_last_used_last%
				{
					index_less_than_next := A_index
					If (countingEachIndex[A_Index+1] > countingEachIndex[A_Index]) {
						; sort
						status_sorted_secondary := 0
						loop_sort_bubble_sletteknapp:
						While, !status_sorted_secondary
						{
							loop_bakover_sletteknapp:
							Loop, %index_less_than_next%
							{
								status_sorted_secondary := 1
								i_bakover := index_last_used_last - A_index +1	; searching backwards
								If ( countingEachIndex[i_bakover+1] > countingEachIndex[i_bakover] ) {
									
									; swap contents at countingEachIndex[]
									hitCounts_temp := countingEachIndex[i_bakover]
									countingEachIndex[i_bakover] := countingEachIndex[i_bakover+1]
									countingEachIndex[i_bakover+1] := hitCounts_temp
									
									; swap contents at orderOfFunctionsSorted[]
									rekkefolge_temp := orderOfFunctionsSorted[i_bakover]
									orderOfFunctionsSorted[i_bakover] := orderOfFunctionsSorted[i_bakover+1]
									orderOfFunctionsSorted[i_bakover+1] := rekkefolge_temp
								}
							}
						}	; while "not sorted"
					}
					Else	; next one is already bigger - skip
						Continue, loop_search_all_decending_sletteknapp
				}	; sort main
				Break
			}
		}
	}	; Loop outer
}


; menu_slett3.bmp  size 18x21	Offset: 21x25	offset-img-coords_bg-color-change: 4x4
sletteknappFinnTrykk_U1() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett3.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+18, funnetY+25, funnetX+4, funnetY+4)
			Return 1
		}
	}
	ImageSearch, funnetX, funnetY, 104, 53, 1282, 125, *44 menu_slett3.bmp
	If (!errorlevel) {	; Large Area Search (LAS)
		x2 := funnetX + 18
		y2 := funnetY + 21
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
}

; menu_slett4.bmp  size 18x16	Offset: 9x8	offset-img-coords_bg-color-change: 2x6
sletteknappFinnTrykk_U2() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett4.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+9, funnetY+8, funnetX+2, funnetY+6)
			Return 1
		}
	}
	ImageSearch, funnetX, funnetY, 104, 53, 1282, 125, *44 menu_slett4.bmp
	If (!errorlevel) {	; Large Area Search (LAS)
		x2 := funnetX + 18
		y2 := funnetY + 16
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
}

; menu_slett2.bmp  size 38x50	Offset: 16x22	offset-img-coords_bg-color-change: 27x25
sletteknappFinnTrykk_U3() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett2.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+16, funnetY+22, funnetX+27, funnetY+25)
			Return 1
		}
	}
	
	ImageSearch, funnetX, funnetY, 396,54, 452,122, *44 menu_slett2.bmp
	If (!errorlevel) {
		x2 := funnetX + 38
		y2 := funnetY + 50
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
	
	ImageSearch, funnetX, funnetY, 104, 53, 1282, 125, *44 menu_slett2.bmp
	If (!errorlevel) {	; Large Area Search (LAS)
		x2 := funnetX + 38
		y2 := funnetY + 50
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
}

; Same image as for sletteknappFinnTrykk_U6 - but another area (2) to search.
sletteknappFinnTrykk_U4() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett7.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+10, funnetY+11, funnetX+7, funnetY+6)
			Return 1
		}
	}
	ImageSearch, funnetX, funnetY, 105,96, 149,124, *44 menu_slett7.bmp
	If (!errorlevel) {
		x2 := funnetX + 14
		y2 := funnetY + 14
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
}

; Same image as for sletteknappFinnTrykk_U6 - but another area (3) to search.
sletteknappFinnTrykk_U5() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett7.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+5, funnetY+5, funnetX+10, funnetY+11)
			Return 1
		}
	}
	ImageSearch, funnetX, funnetY, 888,72, 920,102, *44 menu_slett7.bmp
	If (!errorlevel) {	; normalt avgrenset søk (NAS) - normalt å finne knappen akkurat her (trase)
		x2 := funnetX + 14
		y2 := funnetY + 14
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
}

; This image search is most likely to find the image.
sletteknappFinnTrykk_U6() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett7.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+6, funnetY+6, funnetX+10, funnetY+12)
			Return 1
		}
	}
	
	ImageSearch, funnetX, funnetY, 406,96, 470,124, *44 menu_slett7.bmp
	If (!errorlevel) {	; normalt avgrenset søk (NAS) - normalt å finne knappen akkurat her (normale nettkomponenter)
		x2 := funnetX + 14
		y2 := funnetY + 14
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
	
	; Default menu - button should not exist outside the given areas. Uncomment this to allow searcing on a bigger area.
	/*
	ImageSearch, funnetX, funnetY, 104, 53, 1282, 125, *44 menu_slett7.bmp
	If (!errorlevel) {	; Large Area Search (LAS)
		x2 := funnetX + 14
		y2 := funnetY + 14
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
	*/
}

; menu_slett7b.bmp  size 17x15	Offset: 7x7	offset-img-coords_bg-color-change: 10x13
sletteknappFinnTrykk_U7() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett7b.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
			Return 1
		}
	}
	ImageSearch, funnetX, funnetY, 104, 53, 1282, 125, *44 menu_slett7b.bmp
	If (!errorlevel) {	; Large Area Search (LAS)
		x2 := funnetX + 17
		y2 := funnetY + 15
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
}

; menu_slett8.bmp  size 16x12	Offset: 7x5	offset-img-coords_bg-color-change: 2x5
sletteknappFinnTrykk_U8() {
	Static funnetX, funnetY, x2, y2
	If (funnetX) {	; SMALL SEARCH
		ImageSearch, funnetX, funnetY, funnetX, funnetY, x2, y2, *44 menu_slett8.bmp
		If (!errorlevel) {
			KlikkHoverVentFargeChange(funnetX+7, funnetY+5, funnetX+2, funnetY+5)
			Return 1
		}
	}
	ImageSearch, funnetX, funnetY, 104, 53, 1282, 125, *44 menu_slett8.bmp
	If (!errorlevel) {	; Large Area Search (LAS)
		x2 := funnetX + 16
		y2 := funnetY + 12
		KlikkHoverVentFargeChange(funnetX+7, funnetY+7, funnetX+10, funnetY+13)
		Return 1
	}
}


; View contents in array - gives user an overview of what functions does find image most often.
arrayContentText2(arrayID, counterArray) {
	textOut := "Index`t`tVerdi`t`tTellinger`n"
	For i , Value in arrayID
		textOut .= i . "`t-`t" . Value . "`t-`t" . counterArray[i] . "`n"
	TrayTip, Verdi i tabell,%textOut%
}


; Mouse must hover over a button, and only when some areas of that buttons changes color - it allow the user to click on the button.
; Button in user interface won't respond if mouse click and moves too fast. Also it is not a fixed time delay.
; Throws error when timeout.
KlikkHoverVentFargeChange(klikkeX, klikkeY, fargePixelX, fargePixelY, mousePointerReturns:=1, max_waiting_time:=700) {
	MouseGetPos, musX, musY
	maxTid := A_TickCount + max_waiting_time
	PixelGetColor, button_std_bg_color, fargePixelX, fargePixelY, RGB	; Hente farge1
	MouseMove, klikkeX, klikkeY, 0
	Sleep 50
	Loop, 3000	; Looking for change in pixel value.
	{
		If (A_TickCount > maxTid)
			Throw "Timeout max time"
		PixelGetColor, button_color_possiblechanged, fargePixelX, fargePixelY, RGB
		If (button_color_possiblechanged != button_std_bg_color) {
			Send, {LButton down}
			Sleep 50
			Send, {LButton up}
			Sleep 50
			If mousePointerReturns
				MouseMove, musX, musY, 0
			Return 1
		}
		If A_Index >= 3000
			Throw "Timeout - counting to max"
	}
}

The above function is based on this sample code:
https://autohotkey.com/boards/viewtopic ... 87#p121587

[edit]
Forum gives error message when I tries to upload png images - size of those images is as small as possible - smallest is 14x14 px and biggest is 38x50 px.

Error
Could not upload attachment to ./files/60087_b6fc2f6aadbc21b63611f09ce2994a6b.

[nnnik]

With cancerous I mean that you failed to minimize your code. You failed to compress it.
There is a vastly shorter Version of your code that is also easier to understand.

If I were to summarize your intention:
You have a group of different areas and images that are used to search for the Button delete.
Once the button is found it is clicked ( by your special hover click function )
The search is executed in a way thats very efficient.
In order to make your search very efficient you have implemented these optimisations:
Once you have found the button, you will buffer the results, and only test if the value is still up to date.
The area and image combination that is most likely to find the picture again is executed first.

If I were to summarize your code:
You have a goup of functions that each contain the code for 1 ImageSearch.
If it matches anything will be buffered and henceforth only test whether the value is up to date or not. Once it's not found all buffering is cleared and it searches again. Each function also clicks the found button once called.
These functions are called and ordered by another function by how often they have found the picture.

The first difference between your intention and your code is that instead of grouping the areas and images together in a group construct ( e.g. an Array ) you put each one into a function individually. This is also the main reason as to why your code is so long and unreadeable and also the reason why I dislike it.
The second thing would be the way you buffer your values. You let each function buffer it's value individually where as it makes sense to collect the position and picture combination that has been found before somewhere central (and then test if the button appeared there again).

Code: Select all

class fastSearch ;You could also create functions but I'm better with classes
{
	foundPositions   := []
	searches           := []
	currentPosition := ""

	addSearch( image, area, clickOffset, variation )
	{
		This.searches.Push( {image:image,area:area,searched:0,found:0,clickOffset:clickOffset,variation:variation,foundpositions:[]} )
	}
	
	getClickPosition()
	{
		if ( isObject( This.currentPosition ) && ( This.testPosition( This.currentPosition ) )
			return This.currentPosition.ClickPosition
		For each, Position in This.foundPositions
			if ( This.testPosition( Position ) )
			{
				This.currentPosition := position
				This.sortPositions()
				return Position.ClickPosition
			}
		For each, Area in This.searches
			if isObject( Position := This.testArea( Area ) )
			{
				This.currentPosition := Position
				Loop % This.searches.Length() - each
					This.testAreaForPosition( This.searches[ each + A_Index ], Position )
				This.sortPositions()
				This.sortAreas()
				return Position.ClickPosition
			}
	}
	
	testArea( Area )
	{
		;Do ImageSearch with the parameters from Area ( image, area and variation )
		;If found compare the found positions x and y coordinates with the already found positions and add them if necessary
		;increase "searched" and adjust "found" values of the Position Object you built or loaded
		;if found return a Position Object if not found return ""
		/*
		Position Object needs image, the x and y values, a ClickPosition value ( x+ClickOffset.x , y+ClickOffset.y ) a variation value and searched and found value ( both at 1 )
		*/
	}
	
	testPosition( Position )
	{
		;Do ImageSearch at the exact position with the exact width of the picture from Position and take image and variation ( x, y ,image.w, image.h, image, variation )
		;increase "searched" and adjust "found" values
		;if found return 1 otherwise return 0
	}
	
	testAreaForPosition( Area, Position )
	{
		;test if Position is already a found position of area
		;If it isn't test if they have the same image and then test if position is within area
		;if it is within the area and the same picture but not yet part of the found positions of area then add the Position to Area
		;increase "searched" and adjust "found" values
	}
	
	sortPositions()
	{
		;sort positions according to their found/searched ratio		
	}
	
	sortAreas()
	{
		;sort areas according to their found/searched ratio
	}
}

[Almost_there]

Hi.

nnnik, I greatly appreciate you taking your time and explain. I also think the idea of using a class in this way is very clever, but I need some time to read and try to fully understand, and I think I'll spend some time to rewrite my code, maybe not this week but when I get some spare time.

As far I can see, the idea of your script is that for each search - it buffer up every possible matches. I see forward to test out this approach.

[Almost_there]

Hi.

While still thinking about how to build the script as nnnik have planted a idea, I did get some other idea. Same problem but to make the solution as easy as possible. This one have only four possible images to hit, and I already know that two button types is found nearly every time. The third and fourth is seldom/never used - so therefore my problem is to sort only the two first function calls.

Read: This is another problem, just slightly similar to the first post. In my first post I wanted to sort ALL the functions for imagesearch, while this time I just need to sort between two. I still intend to make a script proposed by nnnik, but I won't spend my work hours for that. Therefore I got onto this easiest solution.

Code: Select all

/*
 
 First to state (boring section):
 The goal of this is to make an _EASY_AS_POSSIBLE_ solution to a problem: How to conduct
 an imagesearch effectively? It is _NOT_ the method that in general gives the best overall result, because
 it can only store (buffer) one location pr function call per image.
 So if we have a situation where the same image (button) often move around to a dosen fixed locations, this solution is just
 not a good one. In my case, a button typically stay in position for a length of time - until the program I work on
 changes module. Therefore this solution works very good - I made a counter that keep track of how often the
 ImageSearch is conducted on the big area (whole ribbon menu) - and it lies around one of ten conducted
 ImageSearch - provided my way of working. For others, this number might change.
 (end of boring section)
 
 Since we don't have the AHK version 2 ability to return whatever number 
 that is returned from whatever statement within an IF() that returns something
 other than 0 - need to pass a variable with ByRef to keep track of the sort order.
 
 There is only the two first function call that has the ability to swap order.
 This is probably the very easiest way to accomplist this sort order result.
 
 The second function may or may not find the image if the first don't - so it would not make any
 sense to move it to the very end of queue. The third and fourth function is much less likely to
 find the associated images so it does not make any sense to move those forward in queue.
 
 This script is very downstripped from the original one, so I cannot promise it hasn't bugs
 because I haven't tested it as it is now.
 
*/

; The function that is ment to being called up.
; Return 1 if image is found - and button is clicked.
imageSearchAndClick() {
	Static search_order := 0	; 0 is start value: 1=defalt button  2=default2 button
	If (search_order < 2) {	; default order from start, tries firstbutton1.
		If !(button_search_click_1(search_order)  ||  button_search_click_1b(search_order)  ||  button_search_click_2()  ||  button_search_click_3() )
			Return 0	; found none
	} Else {	; First and second search is swapped - get slightly faster.
		If !(button_search_click_1b(search_order)  ||  button_search_click_1(search_order)  ||  button_search_click_2()  ||  button_search_click_3() )
			Return 0	; found none
	}
	Return 1
}


/*
	
	The next four functions is (almost) identical except for the image file names and the offset (where shall
	the mouse go to hit center of button?).
	
	The two first function use ByRef - so that the main function would know wich of the two first functions
	did find the image. That is used for determining if the first or second function is to be called first.
	
	The idea behind the functions below is that the image position of successfull imagesearch is stored - so next time
	the same function runs, it search only an area that has same area as the image itself. This way, the image search is
	conducted in far less time that searching the whole area over again.
	If the second call to a function does not result in a successfull imagesearch, it search for the same image
	within the whole search area (a ribbon interface in this case).
	
*/

; NOT SUPPOSED TO BE CALLED DIRECTLY.
button_search_click_1(ByRef search_order_1) {
	Static x1, y1
	button_search_click_1_RepeatSok:
	If (!x1)
		ImageSearch, x1, y1, 49,31, 703,51, button_1.bmp
	Else {
		x2 := x1 + 14
		y2 := y1 + 8
		ImageSearch, x1, y1, x1, y1, x2, y2, button_1.bmp
		If errorlevel
			Goto button_search_click_1_RepeatSok
		Else {
			search_order_1 := 1
			klikkReturnFaster(x1+7, y1+4)
			Return 1
		}
	}
	if !errorlevel {
		search_order_1 := 1
		klikkReturnFaster(x1+7, y1+4)
		Return 1
	}
}


; NOT SUPPOSED TO BE CALLED DIRECTLY.
button_search_click_1b(ByRef search_order_2) {
	Static x1, y1
	button_search_click_1b_RepeatSok:
	If (!x1)
		ImageSearch, x1, y1, 49,31, 703,51, button_1b.bmp
	Else {
		x2 := x1 + 14
		y2 := y1 + 8
		ImageSearch, x1, y1, x1, y1, x2, y2, button_1b.bmp
		If errorlevel
			Goto button_search_click_1b_RepeatSok
		Else {
			search_order_2 := 2
			klikkReturnFaster(x1+7, y1+4)
			Return 1
		}
	}
	if !errorlevel {
		search_order_2 := 2
		klikkReturnFaster(x1+7, y1+4)
		Return 1
	}
}


; NOT SUPPOSED TO BE CALLED DIRECTLY.
button_search_click_2() {
	Static x1, y1
	button_search_click_2_RepeatSok:
	If (!x1)
		ImageSearch, x1, y1, 49,31, 703,51, button_2.bmp
	Else {
		x2 := x1 + 14
		y2 := y1 + 8
		ImageSearch, x1, y1, x1, y1, x2, y2, button_2.bmp
		If errorlevel
			Goto button_search_click_2_RepeatSok
		Else {
			klikkReturnFaster(x1+7, y1+4)
			Return 1
		}
	}
	if !errorlevel {
		klikkReturnFaster(x1+7, y1+4)
		Return 1
	}
}


; NOT SUPPOSED TO BE CALLED DIRECTLY.
button_search_click_3() {
	Static x1, y1
	button_search_click_3_RepeatSok:
	If (!x1)
		ImageSearch, x1, y1, 49,31, 703,51, button_3.bmp
	Else {
		x2 := x1 + 14
		y2 := y1 + 8
		ImageSearch, x1, y1, x1, y1, x2, y2, button_3.bmp
		If errorlevel
			Goto button_search_click_3_RepeatSok
		Else {
			klikkReturnFaster(x1+7, y1+4)
			Return 1
		}
	}
	if !errorlevel {
		klikkReturnFaster(x1+7, y1+4)
		Return 1
	}
}