AutoHotkey Community

Let's help each other out
https://www.autohotkey.com/boards/

WebWriter - By Tank

https://www.autohotkey.com/boards/viewtopic.php?f=6&t=68961

Page 1 of 1

WebWriter - By Tank

Posted: 17 Oct 2019, 13:06
by tank
This still has alot of work to go as to writing a script with frames and dealing with anything that isnt clickable
Hopefully you find it really helpful. basically what it does is allows you to click a web object and it then attempts to build a css selector and write a script using my iwebBrowser2 class.

this code is a prototype and should at this ALPHA stage only be used with great care.

If it helps you please feel free to help me out while i am out of work
Help me out

version .03 Alpha
Corrected tag siblings and other minor corrections

Code: Select all

#NoEnv
winactivate ahk_class IEFrame


^esc:: ExitApp

#IfWinActive, ahk_class IEFrame

    F1::click() ;; click

    F2::value() ;;text or value

    F3::src() ;; image or frame src URI

    F4::checked() ;; is checked true false or blank 

#IfWinActive

elementUnderMouse(ByRef ele){
    getQuerySelector := getQuerySelector(ele := elementFromMouse())
    getQuerySelector = %getQuerySelector%
    return  getQuerySelector
}

writeScript(event){
    WinGetActiveTitle, winTitle
    winTitle := RegExReplace(winTitle, "[^\w\s\-]")
    scriptFile := A_WorkingDir "\" winTitle ".ahk"
    if !FileExist(scriptFile)
        FileAppend,
            ( LTrim
            #Include, lib\iWebBrowser2.ahk
            ie := new iWebBrowser2()
            pwb:=ie.pwb
            while pwb.busy 
                `tSleep, 500
            sleep 2000

            ;; auto generated code below

            ), %scriptFile%
    ;; end if
    if event
        FileAppend, %event%, %scriptFile%
}

value(){
    ret := "", variable := "hovertext" A_Now, cssSelector := elementUnderMouse(ele), element := new IE_HtmlElement(ele), attribute := element.attribute, txt := element.text
    
    if cssSelector && element && attribute
            ret = 
                ( LTrim

                ;; hovered over "%txt%"
                %variable% := ie.pdoc().querySelector("%cssSelector%").%attribute% ;; should be a string

                )
    ObjRelease(ele), ObjRelease(element), writeScript(ret)
}

src(){
    ret := "", variable := "Source" A_Now
    if cssSelector := elementUnderMouse(ele)
        && element := new IE_HtmlElement(ele)
        && element.getSrc()
        && tagName := element.tag
            ret = 
                ( LTrim

                ;; get source of an %tagName% tag
                %variable% := ie.pdoc().querySelector("%cssSelector%").src ;; should be a URI

                )
    ObjRelease(ele), ObjRelease(element), writeScript(ret)
}

checked(){
    ret := "", variable := "isChecked" A_Now
    if cssSelector := elementUnderMouse(ele)
        && element := new IE_HtmlElement(ele)
        && InStr(element.getType(), "checkbox:radio")
        && txt := element.element.parentNode.innerText
            ret = 
                ( LTrim

                /*
                is input checked near 
                "txt"
                */
                %variable% := ie.pdoc().querySelector("%cssSelector%").checked ;; -1 means checked 0 means not checked

                )
        
    
    ObjRelease(ele), ObjRelease(element), writeScript(ret)
}

click(){
    ret := "", variable := isChecked A_Now
    if (cssSelector := elementUnderMouse(ele))
        && (element := new IE_HtmlElement(ele))
        && (txt := element.text)
        ret = 
            ( LTrim

            ;; clicked "%txt%"
            ie.pdoc().querySelector("%cssSelector%").click() ;; execute the click event

            ;; wait for any results to settle
            Sleep, 1000
            while pwb.busy 
                `tSleep, 500
            Sleep, 1000

            )
        
    
    ObjRelease(ele), ObjRelease(element), writeScript(ret)
}

getQuerySelector(ele){
    static iterCount := 0
    try id := ele.id
    if (iterCount++ < 10 && !id)
        path := getQuerySelector(ele.parentNode)
    else iterCount := 0
    return path getTagName(ele)
}

getTagName(ele){
    try tagName := ele.tagName, id := ele.id
    return !id ? siblingList(ele, tagName) : tagName "#" id " "
}

siblingList(ele, tagName){
    tag := tagName
    while ele := ele.previousSibling
        tag .= " + " tagName
    return tag " "
}

elementFromMouse(){
    try
    If (hWnd := hWNDunderMouse(xpos, ypos)) && (pwin := $AccessibleObjectFromWindow(hWnd))
        && (pelt := pwin.document.elementFromPoint(xpos-(xorg:=pwin.screenLeft), ypos-(yorg:=pwin.screenTop)))
    While isFrame(pelt){
        ObjRelease(pwin), pwin :=  ComObj( 9, ComObjQuery( cw := pelt.contentwindow, $IHTMLWindow2 := "{332C4427-26CB-11D0-B483-00C04FD90119}", $IHTMLWindow2 ), 1 ), ObjRelease(cw)
        , xorg+=pelt.getBoundingClientRect.left, yorg+=pelt.getBoundingClientRect.top, ObjRelease(pelt), pelt := pwin.document.elementFromPoint(xpos-xorg, ypos-yorg)
    }
    ObjRelease(pwin)
    Return	pelt
}

hWNDunderMouse( ByRef xpos, ByRef ypos){
    CoordMode, Mouse
    MouseGetPos, xpos, ypos,, hWnd, 3
    return hWnd
}

$AccessibleObjectFromWindow(hIESrv){
    IID_IAccessible := "{618736E0-3C3D-11CF-810C-00AA00389B71}", VarSetCapacity( GUID, 16, 0 ), DllCall("LoadLibrary", "str", "oleacc.dll")
    DllCall( "ole32\CLSIDFromString"
            , "wstr", IID_IAccessible
            , "ptr", &GUID )
    DllCall( "oleacc\AccessibleObjectFromWindow"
            , "Uint", hIESrv
            , "int", -4
            , "ptr", &GUID
            , "UintP", pacc)
    pacc:=ComObj( 9, pacc, 1 )
    pwin := ComObj( 9, ComObjQuery( pacc, $IHTMLWindow2 := "{332C4427-26CB-11D0-B483-00C04FD90119}", $IHTMLWindow2 ), 1 )
    ObjRelease( pacc ), DllCall("FreeLibrary", "str", "oleacc.dll")
    return  pwin
}

isFrame(ele){
    return ele.tagName="IFRAME" || ele.tagName="FRAME"
}

class iWebBrowser2 {

    __new( ByRef sURL = "", ByRef sTitle = "", ByRef iHWND = "",  ByRef sHTML = "" , bVisible = true ) 
        { ;; returns an iwebbrowser2 object
        if !(this.pwb := this.oIE_get(sTitle,iHWND,sURL,sHTML))
            {
            this.pwb := this.oIE_new(sURL,sTitle,iHWND,sHTML,bVisible)
            this.pwb.Navigate(sURL)
            }
            
        return this
        }
    
    oIE_new(ByRef sURL = "about:blank", ByRef sTitle = "", ByRef iHWND = "", ByRef sHTML = "", bVisible = true)
        {
        this.isInstalled()
        this.pwb := ComObjCreate("internetexplorer.application")
        this.pwb.Visible := bVisible
        if sURL
            this.pwb.Navigate(sURL)
        return this.pwb
        }

    oIE_get( ByRef sTitle = "", ByRef iHWND = "", ByRef sURL = "", ByRef sHTML = "" )
        {
        this.isInstalled()
        ;~ this function is pointless if no instance of IE is open
        ;~ one edit you might make is to have this function open IE and maybe go to the home page
        if ( !winexist( "ahk_class IEFrame" ) )
            return false
        
        if sTitle
            this.clean_IE_Title( sTitle ) 
        ;; ok this function should look at all the existing IE instances and build a reference object
        ; List all open Explorer and Internet Explorer windows:
        oIE := Object()
        matches := 0
        
        for window,k in ComObjCreate("Shell.Application").Windows
            if ( "Internet Explorer" = window.Name)
                {
                possiblematch := true

                try pdoc := window.document
                Catch, e
                    while window.busy 
                        Sleep, 500

                if !window.document
                    Continue

                if ( possiblematch && sTitle && !instr( pdoc.title, sTitle ) )
                    possiblematch := false
                
                if ( possiblematch && sHTML && !instr( pdoc.documentelement.outerhtml, sHTML ) )
                    possiblematch := false
                
                if ( possiblematch && sURL && !instr( pdoc.url, sURL ) )
                    possiblematch := false
                
                if ( possiblematch && iHWND > 0 && window.HWND != iHWND )
                    possiblematch := false		
                    
                if ( possiblematch )
                    {
                    ;~ windowsList .= k " => " ( clipboard := window.FullName ) " :: " pdoc.title " :: " pdoc.url "`n"
                    matches++
                    sTitle := pdoc.title
                    sURL := pdoc.url
                    iHWND := window.HWND
                    sHTML := pdoc.documentelement.outerhtml
                    oIE := window
                    }
                ObjRelease( pdoc )
                }
                
        if ( matches > 1 )
            {
            MsgBox, 4112, Too many Matches ,  Please modify your criteria or close some tabs/windows and retry
            return false
            }
        
        return this.pwb := oIE
        }

    FindbyText(needle)
        { ;; returns the element with the text in it  
        try rng:=this.pdoc().body.createTextRange()
        try rng.findText(needle)
		return try rng.parentElement()
	    }

    Activate(pwb) 
        { 
        DllCall("LoadLibrary", "str", "oleacc.dll") 
        HWND:=pwb.HWND
        DetectHiddenWindows, On 
        WinActivate,% "ahk_id " HWND
        WinWaitActive,% "ahk_id " HWND,,5
        ControlGet, hTabBand, hWnd,, TabBandClass1, ahk_class IEFrame
        ControlGet, hTabUI  , hWnd,, DirectUIHWND1, ahk_id %hTabBand% 
        
        VarSetCapacity(CLSID, 16)
        nSize=38
        wString := sString := "{618736E0-3C3D-11CF-810C-00AA00389B71}"
        if(nSize = "")
            nSize:=DllCall("kernel32\MultiByteToWideChar", "Uint", 0, "Uint", 0, "Uint", &sString, "int", -1, "Uint", 0, "int", 0)
        VarSetCapacity(wString, nSize * 2 + 1)
        DllCall("kernel32\MultiByteToWideChar", "Uint", 0, "Uint", 0, "Uint", &sString, "int", -1, "Uint", &wString, "int", nSize + 1)
        DllCall("ole32\CLSIDFromString", "Uint",&wString , "Uint", &CLSID)
        
        If   hTabUI && DllCall("oleacc\AccessibleObjectFromWindow", "Uint", hTabUI, "Uint",-4, "Uint", &CLSID , "UintP", pacc)=0 
            { 
            pacc := ComObject(9, pacc, 1), ObjAddRef(pacc)
            Loop, %   pacc.accChildCount 
                If   paccChild:=pacc.accChild(A_Index) 
                    If   paccChild.accRole(0+0) = 0x3C 
                        { 
                        paccTab:=paccChild 
                        Break 
                        } 
                    Else   ObjRelease(paccChild) 
            ObjRelease(pacc) 
            } 
        If   pacc:=paccTab 
            { 
            Loop, %   pacc.accChildCount
                If   paccChild:=pacc.accChild(A_Index) 
                    If   paccChild.accName(0+0) = sTitle   
                        { 
                        ObjRelease(pwb)
                        paccChild.accDoDefaultAction(0)
                        ObjRelease(paccChild) 
                        Break 
                        } 
                    Else   ObjRelease(paccChild) 
            ObjRelease(pacc) 
            }  
        WinActivate,% sTitle
        } 


    isInstalled()
        {
        Static IE_path
        
        ;; find where windows believes IE is installed
        ;; certain corp installs may have this in other than expected folders
        if !IE_path
            RegRead, IE_path, HKLM, SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\IEXPLORE.EXE
        ;~ MsgBox % IE_path
        ;; Perhaps policies prevent reading this key
        if ( ErrorLevel || !IE_path )
            IE_path := "C:\Program Files\Internet Explorer\iexplore.exe"
        
        ;; make sure it installed
        if !FileExist( IE_path )
            {
            MsgBox, 4112, Internet Explorer Not Found, IE does not appear to be installed`nCannot continue `nClick OK to Exit!!!
            ExitApp
            }
        } 

    pDoc()
        {
        return this.IHTMLWindow2_from_IWebDOCUMENT( this.pwb.document ).document
        }

    clean_IE_Title( ByRef sTitle = "" ) 
        {
        return sTitle := RegExReplace( sTitle ? sTitle : this.active_IE_Title(), this.IE_Suffix() "$", "" )
        }

    IE_Suffix() 
        {
        static sIE_Suffix
        if !sIE_Suffix
            {
            ;; HKEY_CURRENT_USER\Software\Microsoft\Internet Explorer\Main
            RegRead, sIE_Suffix, HKCU, Software\Microsoft\Internet Explorer\Main, Window Title ;, Windows Internet Explorer,
            sIE_Suffix := " - " sIE_Suffix
            }
        return sIE_Suffix
        }

    active_IE_Title() ;; returns the title of the topmost browser if exists from the stack
        {
        sTitle := "NO IE Window Open"
        if winexist( "ahk_class IEFrame" )
            {
            titlematchMode := A_TitleMatchMode
            titlematchSpeed := A_TitleMatchModeSpeed
            SetTitleMatchMode, 2	
            SetTitleMatchMode, Slow
            WinGetTitle, sTitle, %sIE_Suffix% ahk_class IEFrame
            SetTitleMatchMode, %titlematchMode%	
            SetTitleMatchMode, %titlematchSpeed%
            }
        return RegExReplace( sTitle, this.IE_Suffix() "$", "" )
        }
        
        
        
    IHTMLWindow2_from_IWebDOCUMENT( IWebDOCUMENT )
        {
        static IID_IHTMLWindow2 := "{332C4427-26CB-11D0-B483-00C04FD90119}"  ; IID_IHTMLWindow2
        return ComObj(9,ComObjQuery( IWebDOCUMENT, IID_IHTMLWindow2, IID_IHTMLWindow2),1)
        }

    IWebDOCUMENT_from_IWebDOCUMENT( IWebDOCUMENT ) ;bypasses certain security issues
        {
        return this.IHTMLWindow2_from_IWebDOCUMENT( IWebDOCUMENT ).document
        }

    IWebBrowserApp_from_IWebDOCUMENT( IWebDOCUMENT )
        {
        static IID_IWebBrowserApp := "{0002DF05-0000-0000-C000-000000000046}"  ; IID_IWebBrowserApp
        return ComObj(9,ComObjQuery( this.IHTMLWindow2_from_IWebDOCUMENT( IWebDOCUMENT ), IID_IWebBrowserApp, IID_IWebBrowserApp),1)
        }

    IWebBrowserApp_from_Internet_Explorer_Server_HWND( hwnd, Svr#=1 ) 
        {               ;// based on ComObjQuery docs
        static msg := DllCall( "RegisterWindowMessage", "str", "WM_HTML_GETOBJECT" )
            , IID_IWebDOCUMENT := "{332C4425-26CB-11D0-B483-00C04FD90119}"
        
        SendMessage msg, 0, 0, Internet Explorer_Server%Svr#%, ahk_id %hwnd%
        
        if (ErrorLevel != "FAIL") 
            {
            lResult := ErrorLevel
            VarSetCapacity( GUID, 16, 0 )
            if DllCall( "ole32\CLSIDFromString", "wstr", IID_IWebDOCUMENT, "ptr", &GUID ) >= 0 
                {
                DllCall( "oleacc\ObjectFromLresult", "ptr", lResult, "ptr", &GUID, "ptr", 0, "ptr*", IWebDOCUMENT )
                return  this.IWebBrowserApp_from_IWebDOCUMENT( IWebDOCUMENT )
                }
            }
        }
}

class IE_HtmlElement {


    __new(ele){
        this.element    := ele
        this.tag        := this.getTag()
        this.type       := this.getType()
        this.checked    := this.getChecked()
        this.text       := this.getText()
        this.src        := this.getSrc()
    }

    getText(){
        if (instr(this.getTag(), "input:"))
            txt := this.element.value, this.attribute := "value"
        else if (instr(this.getTag(), "select:"))
            txt := this.element.selectedText, this.attribute := "selectedText"
        else txt := this.element.innerText, this.attribute := "innerText"
        ToolTip, %txt%
        return txt
    }
    
    getTag(){
        return this.element.tagname
    }
    
    getType(){
        if (instr(this.getTag(), "input:"))
            return this.element.type
    }
    
    getChecked(){
        if (instr(this.getType(), "checkbox:radio"))
            return this.element.checked
    }
    
    getSrc(){
        if (instr(this.getTag(), "image:frame"))
            return this.element.src
    }

    setText(txt){
        
    }
}


version .02 alpha posted in op
corrected issue identified by gregster
added methods for
  • click
  • get text that works on both inputs and regular elements
  • get status clicked or not of radio and checkbox
  • get source of images and frames
Spoiler
v0.01
Spoiler

Re: WebWriter - By Tank

Posted: 17 Oct 2019, 21:46
by guest3456
can you give an example of what you'd use this for?

Re: WebWriter - By Tank

Posted: 18 Oct 2019, 02:42
by gregster
Thanks tank, looks interesting.
guest3456, As I understood it, you can sort of record some action(s) like a click on a certain website element in a script - and then replay these actions by simply running this script (and including the iWebBrowser2 library).

Had only a quick go so far, and created a script with your WebWriter. It created a css selector, but there seemed to be some problem and code missing (although I made the iWebBrowser2 library available).

After glancing at the source, I think that at least this FileCopy command below should be replaced with FileAppend, as the following code from the first continuation section was completely missing from the created script:

Code: Select all

; [...]
 if !FileExist(scriptFile){
        FileCopy,
(
#Include, lib\iWebBrowser2.ahk
ie := new iWebBrowser2()
; [...]
Will have another go later!

Re: WebWriter - By Tank

Posted: 28 Oct 2019, 18:26
by tank
version .02 alpha posted in op
corrected issue identified by gregster
added methods for
  • click
  • get text that works on both inputs and regular elements
  • get status clicked or not of radio and checkbox
  • get source of images and frames

Re: WebWriter - By Tank

Posted: 30 Oct 2019, 08:06
by burque505
@tank, thanks for this!
Regards,
burque505

Re: WebWriter - By Tank

Posted: 04 Nov 2019, 21:37
by tank
version .03 Alpha
Corrected tag siblings and other minor corrections

Re: WebWriter - By Tank

Posted: 05 Nov 2019, 20:36
by SOTE
gregster wrote: ↑
18 Oct 2019, 02:42
 Thanks tank, looks interesting.
guest3456, As I understood it, you can sort of record some action(s) like a click on a certain website element in a script - and then replay these actions by simply running this script (and including the iWebBrowser2 library).
It's fantastic that Tank came out with this type of recorder script for web browser automation.

Re: WebWriter - By Tank

Posted: 06 Nov 2019, 00:37
by rommmcek
This is very fine script! I wish I was able to use all it's potential!

P.s.: Here is my contribution. Modified function elementFromMouse() should enable work at any Zoom:

Code: Select all

elementFromMouse(){
    try
    If (hWnd := hWNDunderMouse(xpos, ypos)) && (pwin := $AccessibleObjectFromWindow(hWnd), Zm:=pwin.document.parentwindow.screen.deviceXDPI/96)
        && (pelt := pwin.document.elementFromPoint(xpos/Zm-(xorg:=pwin.screenLeft), ypos/Zm-(yorg:=pwin.screenTop)))
    While isFrame(pelt){
        ObjRelease(pwin), pwin :=  ComObj( 9, ComObjQuery( cw := pelt.contentwindow, $IHTMLWindow2 := "{332C4427-26CB-11D0-B483-00C04FD90119}", $IHTMLWindow2 ), 1 ), ObjRelease(cw)
        , xorg+=pelt.getBoundingClientRect.left, yorg+=pelt.getBoundingClientRect.top, ObjRelease(pelt), pelt := pwin.document.elementFromPoint(xpos-xorg, ypos-yorg)
    }
    ObjRelease(pwin)
    Return	pelt
}

Re: WebWriter - By Tank

Posted: 01 Dec 2019, 02:56
by SOTE
I suggest to clarify and post the link for iWebBrowser2.ahk. There seems to be various versions and similarly named files and projects. Would be better if everyone were using the same files to test the program out.

Re: WebWriter - By Tank

Posted: 13 May 2020, 04:37
by BNOLI
No further news/reaction from Irving, TX ? :)

Re: WebWriter - By Tank

Posted: 13 May 2020, 05:10
by malcev
The code is dirty.
For example: here We should change uint to ptr and should not release pacc.
Also this is not correct DllCall("FreeLibrary", "str", "oleacc.dll").

Code: Select all

$AccessibleObjectFromWindow(hIESrv){
    IID_IAccessible := "{618736E0-3C3D-11CF-810C-00AA00389B71}", VarSetCapacity( GUID, 16, 0 ), DllCall("LoadLibrary", "str", "oleacc.dll")
    DllCall( "ole32\CLSIDFromString"
            , "wstr", IID_IAccessible
            , "ptr", &GUID )
    DllCall( "oleacc\AccessibleObjectFromWindow"
            , "Uint", hIESrv
            , "int", -4
            , "ptr", &GUID
            , "UintP", pacc)
    pacc:=ComObj( 9, pacc, 1 )
    pwin := ComObj( 9, ComObjQuery( pacc, $IHTMLWindow2 := "{332C4427-26CB-11D0-B483-00C04FD90119}", $IHTMLWindow2 ), 1 )
    ObjRelease( pacc ), DllCall("FreeLibrary", "str", "oleacc.dll")
    return  pwin
}
All times are UTC-05:00
Page 1 of 1
Powered by phpBB® Forum Software © phpBB Limited
https://www.phpbb.com/