Internet Explorer get element under cursor (show borders, show text) (any zoom percentage)

[jeeswg]

- I've tried creating my own examples of frames within frames e.g. via examples here:
How to Use Frames Tags in HTML »
https://html.com/frames/
However, they look like frames within frames, but I'm not sure that they are actually frames within frames cf. your website.
- If you can give me a link to your website, or send me a saved htm file, or an edited htm file to remove any personal data, or start with the html there and simplify it to create something with the same structure, then I can fix the script.
- If I can get a good html example, then I can fix the script. I'm quite willing to work on this. Until then, I've done as much as I can do.
- I've investigated the maths more fully, so the situation is clearer.
- Some points to mention are frames v. iframes, frames v. framesets, and to check which your website is.
- Did you manage to fix the script yourself? Cheers.

[TheDewd]

Well if there are more levels, you should remove the line 'break'. But I don't know if it will work correctly. If you provide some html examples that have frames within frames, I could experiment. I might try and create such an example myself, so that ultimately my script can handle frames within frames.

Jeeswg: I created an example for you this morning. Please download the HTML files at the following link
https://drive.google.com/open?id=1KxEhW ... WUU954NYJl

Your suggestion to remove the Break did work! Would there be any negative side-effects from removing it? Performance-wise, etc?

[jeeswg]

The break was something I accidentally left in while testing, to prevent an infinite loop, or something like that. I will check the example, thanks.

[TheDewd]

I thought it was behaving correctly when I tested it yesterday, however I now see an issue.

The coordinates are being offset by the size of surrounding frames.

For example, the height of the top frame is offsetting the coordinates in the bottom frame by the exact height of the top frame.

Resizing the top frame to make it shorter is directly affecting the placement of JEE_Borders() in the bottom frame.

Otherwise, removing the Break did work, but I'm not sure how to ignore the size of the other frames. Need to somehow reset the top & left coordinates for each iFrame level, I guess...

[TheDewd]

I finally fixed the offset issue I was experiencing for the iFrames. The website I use only goes to 2 levels deep, so the following changes are working now!!

I changed the following:

FROM:

Code: Select all

oElt2 := oWin.document.elementFromPoint(vMouseX4, vMouseY4)

TO:

Code: Select all

If (A_Index > 1) {
    oElt2 := oWin.document.elementFromPoint(vMouseX4-vCtlX, vMouseY4-vCtlY)
} Else {
    oElt2 := oWin.document.elementFromPoint(vMouseX4, vMouseY4)
}

Code: Select all

#SingleInstance, Force

#IfWinActive, ahk_class IEFrame
q:: ;internet explorer - get element under cursor
;show borders around, and get text from, element under cursor
;regardless of zoom percentage of 'Internet Explorer_Server' control
;it could potentially be incorporated into iWB2 Learner
IID_IWebBrowserApp := "{0002DF05-0000-0000-C000-000000000046}"
IID_IHTMLWindow2 := "{332C4427-26CB-11D0-B483-00C04FD90119}"

CoordMode, Mouse, Screen
MouseGetPos, vMouseX, vMouseY,, hCtl, 3
WinGetClass, vCtlClass, % "ahk_id " hCtl
if !(vCtlClass == "Internet Explorer_Server")
	return
if !(oDoc := ComObject(9,ComObjQuery(Acc_ObjectFromWindow(hCtl),IID_IHTMLWindow2,IID_IHTMLWindow2),1).document)
	return
oWin := ComObject(9,ComObjQuery(oDoc,IID_IHTMLWindow2,IID_IHTMLWindow2),1)
oWB := ComObject(9,ComObjQuery(oWin,IID_IWebBrowserApp,IID_IWebBrowserApp),1)

vLogicalXDPI := oWB.document.parentWindow.screen.logicalXDPI
vDeviceXDPI := oWB.document.parentWindow.screen.deviceXDPI
vZoomRatio := vDeviceXDPI / vLogicalXDPI

;vWinX := oWin.screenLeft, vWinY := oWin.screenTop
;WinGetPos, vWinX2, vWinY2, vWinW2, vWinH2, ahk_class IEFrame
WinGetPos, vCtlX, vCtlY, vCtlW, vCtlH, ahk_id %hCtl%
vMouseX2 := (vMouseX-vCtlX)*(1/vZoomRatio)
vMouseY2 := (vMouseY-vCtlY)*(1/vZoomRatio)

oElt := oWin.document.elementFromPoint(vMouseX2, vMouseY2)

vOffsetX := 0, vOffsetY := 0
while (oElt.tagName = "IFRAME") || (oElt.tagName = "FRAME")
{
	ObjRelease(oWin)
	oWin := ComObject(9,ComObjQuery(oElt2:=oElt.contentwindow,IID_IHTMLWindow2,IID_IHTMLWindow2),1)
	ObjRelease(oElt2)
	oRect := oElt.getBoundingClientRect()
	vOffsetX += (oRect.left * vZoomRatio)
	vOffsetY += (oRect.top * vZoomRatio)
	vMouseX4 := vMouseX2 - oRect.left
	vMouseY4 := vMouseY2 - oRect.top
	;oElt2 := oWin.document.elementFromPoint(vMouseX4, vMouseY4)
    
    /* TheDewd Fix */
    If (A_Index > 1) {
        oElt2 := oWin.document.elementFromPoint(vMouseX4-vCtlX, vMouseY4-vCtlY)
    } Else {
        oElt2 := oWin.document.elementFromPoint(vMouseX4, vMouseY4)
    }
    
	ObjRelease(oElt)
	oElt := oElt2
}

oRect := oElt.getBoundingClientRect()
vEltX := vCtlX + vOffsetX + (oRect.left * vZoomRatio)
vEltY := vCtlY + vOffsetY + (oRect.top * vZoomRatio)
vEltW := (oRect.right - oRect.left) * vZoomRatio
vEltH := (oRect.bottom - oRect.top) * vZoomRatio

JEE_Borders(vEltX, vEltY, vEltW, vEltH, 1000, "FFFF00")
MsgBox % oElt.tagName "`r`n" oElt.innerText
oDoc := oWin := oWB := oElt := oElt2 := oRect := ""
return
#IfWinActive
;==================================================

JEE_Borders(vPosX, vPosY, vPosW=0, vPosH=0, vTime=1000, vCol="FFFF00")
{
CoordMode, Pixel, Screen
vBorderH := 5 ;height of border
vBorderW := 5 ;width of border
SplashImage, 1:, % "B W" vPosW+(2*vBorderW) "H" vBorderH "X" vPosX-vBorderW "Y" vPosY-vBorderH "Cw" vCol,,, SP1 ;top
SplashImage, 2:, % "B W" vPosW+(2*vBorderW) "H" vBorderH "X" vPosX-vBorderW "Y" vPosY+vPosH "Cw" vCol,,, SP2 ;bottom
SplashImage, 3:, % "B W" vBorderW "H" vPosH+(2*vBorderH) "X" vPosX-vBorderW "Y" vPosY-vBorderH "Cw" vCol,,, SP3 ;left
SplashImage, 4:, % "B W" vBorderW "H" vPosH+(2*vBorderH) "X" vPosX+vPosW "Y" vPosY-vBorderH "Cw" vCol,,, SP4 ;right
Sleep % vTime
SplashImage, 1: Off
SplashImage, 2: Off
SplashImage, 3: Off
SplashImage, 4: Off
Return
}

Acc_ObjectFromWindow(hWnd, idObject := -4) {
    Static h

    If !(h) {
        h := DllCall("LoadLibrary", "Str", "Oleacc.dll", "Ptr")
    }

    If !(DllCall("Oleacc.dll\AccessibleObjectFromWindow", "Ptr", hWnd, "UInt", idObject &= 0xFFFFFFFF, "Ptr", -VarSetCapacity(IID, 16) + NumPut((idObject == 0xFFFFFFF0 ? 0x46000000000000C0 : 0x719B3800AA000C81), NumPut((idObject == 0xFFFFFFF0 ? 0x0000000000020400 : 0x11CF3C3D618736E0), IID, "Int64"), "Int64"), "Ptr*", pAcc)) {
        return ComObjEnwrap(9, pAcc, 1)
    }
}

[jeeswg]

This is a copy of TheDewd's example zip from the 13 Nov 2017 post. The original file was called 'Example.zip', I renamed it to 'NestedFramesExample.zip'.

[jeeswg]

- I finally had a chance to get back to the problem of nested frames, and frame/element coordinates, but it's still giving me trouble.
- There are 3 goals I would like to achieve:
-- get frame position
-- get element position
-- convert cursor position to a position that the WB object can handle
- I haven't really looked at the last 2 again yet. I want to understand getting the frame position first.

- This script tries to get the top-left coordinates of a frame and put a 300x300 box on it. It works fine at 100% zoom, and it's working on one or two frames on the Example.htm example from the zip above, at any zoom, but it gets one wrong slightly. I'm not really sure why it's going wrong, in case anybody can come up with some ideas.
- Btw the AutoHotkey documentation now uses frames, so it can be used for testing.
- Btw I've changed the JEE_Borders function to use 0xABCDEF (or 11259375) instead of "ABCDEF" for specifying RGB colour values. I.e. dec/hex numbers and not a 6-character string. A copy of the updated function is provided here.

Code: Select all

;[WBGet function]
;WBGet function - AutoHotkey Community
;https://autohotkey.com/boards/viewtopic.php?f=6&t=39869

;[Example.htm]
;Internet Explorer get element under cursor (show borders, show text) (any zoom percentage) - Page 2 - AutoHotkey Community
;https://autohotkey.com/boards/viewtopic.php?f=6&t=29458&p=212871#p212871

^=:: ;internet explorer - zoom in/out
^-:: ;internet explorer - zoom in/out
if InStr(A_ThisHotkey, "-")
	SendInput, ^{WheelDown} ;zoom out
else
	SendInput, ^{WheelUp} ;zoom in
Sleep, 100
WinGet, hWnd, ID, A
oWB := WBGet("ahk_id " hWnd)
vLogicalXDPI := oWB.document.parentWindow.screen.logicalXDPI
vDeviceXDPI := oWB.document.parentWindow.screen.deviceXDPI
vZoomRatio := vDeviceXDPI / vLogicalXDPI
oWB := ""
MsgBox, % Round(vZoomRatio*100) "%"
return

q:: ;internet explorer - draw temporary borders around frames
WinGet, hWnd, ID, A
oWB := WBGet("ahk_id " hWnd)

vLogicalXDPI := oWB.document.parentWindow.screen.logicalXDPI
vDeviceXDPI := oWB.document.parentWindow.screen.deviceXDPI
vZoomRatio := vDeviceXDPI / vLogicalXDPI

vWinW := vWinH := 300

vWinX1 := oWB.document.parentWindow.screenLeft
vWinY1 := oWB.document.parentWindow.screenTop
vNum := 1 ;note: frames are 0-based, so frame '1' is the 2nd frame
vCountFrame := oWB.document.parentWindow.frames.length
if (vNum < vCountFrame)
	oWB := ComObject(9, ComObjQuery(oWB.document.parentWindow.frames[vNum], "{332C4427-26CB-11D0-B483-00C04FD90119}", "{332C4427-26CB-11D0-B483-00C04FD90119}"), 1)
oWB1 := oWB

vWinX2 := oWB.document.parentWindow.screenLeft
vWinY2 := oWB.document.parentWindow.screenTop
vNum := 1
vCountFrame := oWB.document.parentWindow.frames.length
if (vNum < vCountFrame)
	oWB := ComObject(9, ComObjQuery(oWB.document.parentWindow.frames[vNum], "{332C4427-26CB-11D0-B483-00C04FD90119}", "{332C4427-26CB-11D0-B483-00C04FD90119}"), 1)
oWB2 := oWB

vWinX3 := oWB.document.parentWindow.screenLeft
vWinY3 := oWB.document.parentWindow.screenTop
oWB3 := oWB

vSleep := 500, vColRGB := 0xFFFF00
JEE_Borders(vWinX1, vWinY1, vWinW, vWinH, vSleep, vColRGB)
JEE_Borders(vWinX2, vWinY2, vWinW, vWinH, vSleep, vColRGB)
JEE_Borders(vWinX3, vWinY3, vWinW, vWinH, vSleep, vColRGB)

vOutput := Round(vZoomRatio*100) "%`r`n`r`n"
Loop, 3
	vOutput .= vWinX%A_Index% " " vWinY%A_Index% "`r`n"

vWinX1 *= vZoomRatio**1
vWinX2 *= vZoomRatio**2
vWinX3 *= vZoomRatio**3
vWinY1 *= vZoomRatio**1
vWinY2 *= vZoomRatio**2
vWinY3 *= vZoomRatio**3

Loop, 3
{
	vWinX%A_Index% := Round(vWinX%A_Index%)
	, vWinY%A_Index% := Round(vWinY%A_Index%)
}

JEE_Borders(vWinX1, vWinY1, vWinW, vWinH, vSleep, vColRGB)
JEE_Borders(vWinX2, vWinY2, vWinW, vWinH, vSleep, vColRGB)
JEE_Borders(vWinX3, vWinY3, vWinW, vWinH, vSleep, vColRGB)

vOutput .= "`r`n"
Loop, 3
	vOutput .= vWinX%A_Index% " " vWinY%A_Index% "`r`n"

MsgBox, % vOutput
return

;==================================================

; ;e.g.
; q:: ;draw borders around the active window/control
; hWnd := WinGetID("A")
; vCtlClassNN := ControlGetFocus("ahk_id " hWnd)
; hWnd := ControlGetHwnd(vCtlClassNN, "ahk_id " hWnd)
; WinGetPos(vCtlX, vCtlY, vCtlW, vCtlH, "ahk_id " hWnd)
; JEE_Borders(vCtlX, vCtlY, vCtlW, vCtlH)
; return

JEE_Borders(vPosX, vPosY, vPosW:=0, vPosH:=0, vTime:=1000, vColRGB:=0xFFFF00, vBdrW:=5, vBdrH:=5)
{
	;based on example at the very bottom of:
	;WinSet
	;https://autohotkey.com/docs/commands/WinSet.htm
	static vWinClass := "AHKBordersClass"
	;static vFunc := A_ThisFunc "WndProc" ;doesn't work
	;static pWndProc := RegisterCallback(vFunc, "F")
	static pWndProc
	static vPIs64 := (A_PtrSize=8)
	static vSize := vPIs64?80:48

	if !pWndProc
		pWndProc := RegisterCallback(A_ThisFunc "WndProc", "F")
	VarSetCapacity(WNDCLASSEX, vSize, 0)
	NumPut(vSize, &WNDCLASSEX, 0, "UInt") ;cbSize
	NumPut(pWndProc, &WNDCLASSEX, 8, "Ptr") ;lpfnWndProc
	vColBGR := ((0xFF & vColRGB) << 16) + (0xFF00 & vColRGB) + ((0xFF0000 & vColRGB) >> 16)
	hBrush := DllCall("gdi32\CreateSolidBrush", UInt,vColBGR, Ptr)
	NumPut(hBrush, &WNDCLASSEX, vPIs64?48:32, "Ptr") ;hbrBackground
	NumPut(&vWinClass, &WNDCLASSEX, vPIs64?64:40, "Ptr") ;lpszClassName
	DllCall("user32\RegisterClassEx", Ptr,&WNDCLASSEX, UShort)

	vPosX -= vBdrW, vPosY -= vBdrH
	vPosW += vBdrW*2, vPosH += vBdrH*2

	vDHW := A_DetectHiddenWindows
	;DetectHiddenWindows("On")
	DetectHiddenWindows, On
	;WS_POPUP := 0x80000000
	;WS_EX_TOOLWINDOW := 0x80 ;WS_EX_TOPMOST := 0x8
	vWinText := "", vWinStyle := 0x80000000, vWinExStyle := 0x88
	hWnd := DllCall("user32\CreateWindowEx", UInt,vWinExStyle, Str,vWinClass, Str,vWinText, UInt,vWinStyle, Int,vPosX, Int,vPosY, Int,vPosW, Int,vPosH, Ptr,0, Ptr,0, Ptr,0, Ptr,0, Ptr)
	vBdrL := vBdrR := vBdrW
	vBdrT := vBdrB := vBdrH
	oArray := [vPosW, vPosH, vBdrL, vPosW-vBdrR, vBdrT, vPosH-vBdrB]
	vRegion := Format("0-0 {1:}-0 {1:}-{2:} 0-{2:} 0-0" " {3:}-{5:} {4:}-{5:} {4:}-{6:} {3:}-{6:} {3:}-{5:}", oArray*)
	;WinSetRegion(vRegion, "ahk_id " hWnd)
	;WinShow("ahk_id " hWnd)
	;DetectHiddenWindows(vDHW)
	WinSet, Region, % vRegion, % "ahk_id " hWnd
	WinShow, % "ahk_id " hWnd
	DetectHiddenWindows, % vDHW

	;Sleep(vTime)
	Sleep, % vTime
	DllCall("user32\DestroyWindow", Ptr,hWnd)
}

JEE_BordersWndProc(hWnd, uMsg, wParam, lParam)
{
	return DllCall("user32\DefWindowProc", Ptr,hWnd, UInt,uMsg, UPtr,wParam, Ptr,lParam, Ptr)
}

;==================================================

[EDIT:] Changed JEE_Borders to make it work in AHK v1.

[jeeswg]

Here's some code plus 3 nested htm examples, if people want to play around with this, and perhaps work out the logic to get the last border splashes in the right places.

ahk test IE zoom.zip

(22.21 KiB) Downloaded 139 times

There's also this important question that needs to be resolved.
Internet Explorer: is frame topmost - AutoHotkey Community
https://autohotkey.com/boards/viewtopic.php?f=5&t=63441