Windows:reading screen contents with mouse

[afk]

What is involved in reading the contents (text) of a
screen in Windows by using the mouse ? I would
like to be able to pick off the word beneath the
mouse pointer and move it to a buffer.
Thanks in advance for any attention to this.
afk

[Timbo]

Your requirements are a bit vague, so I am suspecting the answers (and especially mine) will be vague as well.  Since you speak of a window, I am going to assume you don't mean a console application (could be wrong, but guesses tend to be that way).  Do you have the source of the application in question?

Knowing the font width/height of the text in question, and knowing the position of the mouse inside the client area you can then calculate where inside the drawn text the mouse is pointing to.

More info, please.

[afk]

Hi Timbo,
Thanks for the quick reply. Sorry for my slow response. I
thought my profile had me set up to be emailed about
any action here so I didn't check back until now.

The application would be a Win 2k GUI window written in
assembly using either MASM or NASM.

This GUI would share a portion of the screen with a web
browser screen.

My goal is to be able to load a screen of text into the web
browser and to move the mouse cursor over any area within
the browser page and copy the word under the cursor into
either the clipboard or an application buffer with just a click.

Determining the screen location of the cursor is no problem.

What I don't understand is how to access the screen buffer
and whether Windows stores a text screen as a bitmap or
as chars.

Thanks
afk

[Timbo]

I think you want to write a browser plugin then.  You cannot make an assumption about what the browser is displaying and it would be simply easier to have the browser feed it to you.  The alternatives (including API hooking) open up a rather large can of worms.

[afk]

Thanks for the help Timbo. I'll approach it by trying to latch onto
the browser and leave the can of 'hook' worms unopened ;-)

afk

[MichaelW]

Why not just select the word by dragging the mouse over it, right click, and select copy?

[Tedd]

Because that's not automated - which defeats the purpose of making a bot :P

[afk]

Why not just select the word by dragging the mouse over it, right click, and select copy?

Because that's not automated - which defeats the purpose of making a bot :P

Yes. That's it. I want to remove the annoyance of multiple user steps if I can.
Thanks guys.
afk

[BasilYercin]

hi why not using IAccessbile in order to read screen contents with your mouse position...

Code Select Expand


.486
.MODEL FLAT, STDCALL
option casemap:none

include \masm32\include\windows.inc
include \masm32\include\kernel32.inc
include \masm32\include\user32.inc
include \masm32\include\gdi32.inc
include \masm32\include\ole32.inc
include \masm32\include\oleacc.inc
include \masm32\include\oleaut32.inc
include \masm32\include\masm32.inc

include \masm32\COM\include\oaidl.inc
include \masm32\COM\include\colib.inc
include \masm32\COM\include\L.inc
include \masm32\COM\include\ACTIVSCP.inc    ; NEW colib file

includelib \masm32\lib\kernel32.lib
includelib \masm32\lib\user32.lib
includelib \masm32\lib\gdi32.lib
includelib \masm32\lib\oleacc.lib
includelib \masm32\lib\oleaut32.lib
includelib \masm32\COM\colib\colib.lib
includelib \masm32\lib\masm32.lib

includelib \masm32\lib\ole32.lib
includelib \masm32\lib\oleaut32.lib
includelib \masm32\lib\COMCTL32.LIB

sIID_IAccessible                  TEXTEQU <{618736e0H, 03c3dH, 011cfH, {081H, 00cH, 000H, 0aaH, 000H, 038H, 09BH, 071H}}>

IAccessible_get_accParentProto                 typedef proto :DWORD, :DWORD
IAccessible_get_accChildCountProto             typedef proto :DWORD, :DWORD
IAccessible_get_accChildProto                  typedef proto :DWORD, :WORD, :DWORD  ,:DWORD ,:DWORD ,:DWORD  ; ?
IAccessible_get_accNameProto                   typedef proto :DWORD, :DWORD, :DWORD ,:DWORD ,:DWORD ,:DWORD  ; Ok
IAccessible_get_accRoleProto                   typedef proto :DWORD, :DWORD, :DWORD
IAccessible_get_accValueProto                  typedef proto :DWORD, :DWORD, :DWORD ,:DWORD ,:DWORD ,:DWORD
IAccessible_get_accParentVal                   typedef ptr IAccessible_get_accParentProto
IAccessible_get_accChildCountVal               typedef ptr IAccessible_get_accChildCountProto
IAccessible_get_accChildVal                    typedef ptr IAccessible_get_accChildProto
IAccessible_get_accNameVal                     typedef ptr IAccessible_get_accNameProto
IAccessible_get_accValueVal                    typedef ptr IAccessible_get_accValueProto
IAccessible_get_accDescriptionVal              typedef ptr IAccessible_get_accDescriptionProto
IAccessible_get_accRoleVal                    typedef ptr IAccessible_get_accRoleProto
IAccessible_get_accStateVal                    typedef ptr IAccessible_get_accStateProto
IAccessible_get_accHelpVal                    typedef ptr IAccessible_get_accHelpProto
IAccessible_get_accHelpTopicVal                typedef ptr IAccessible_get_accHelpTopicProto
IAccessible_get_accKeyboardShortcutVal        typedef ptr IAccessible_get_accKeyboardShortcutProto
IAccessible_get_accFocusVal                    typedef ptr IAccessible_get_accFocusProto
IAccessible_get_accSelectionVal                typedef ptr IAccessible_get_accSelectionProto
IAccessible_get_accDefaultActionVal            typedef ptr IAccessible_get_accDefaultActionProto
IAccessible_accSelectVal                      typedef ptr IAccessible_accSelectProto
IAccessible_accLocationVal                    typedef ptr IAccessible_accLocationProto
IAccessible_accNavigateVal                    typedef ptr IAccessible_accNavigateProto
IAccessible_accHitTestVal                      typedef ptr IAccessible_accHitTestProto
IAccessible_accDoDefaultActionVal              typedef ptr IAccessible_accDoDefaultActionProto
IAccessible_put_accNameVal                    typedef ptr IAccessible_put_accNameProto
IAccessible_put_accValueVal                    typedef ptr IAccessible_put_accValueProto


_vtIAccessible MACRO CastName:REQ
    ; IDispatch methods
    _vtIDispatch CastName
    &CastName&_accParent                  comethod2 ?
    &CastName&_get_accChildCount          comethod2 ?
    &CastName&_get_accChild               comethod6 ?
    &CastName&_get_accName                comethod6 ?
    &CastName&_get_accValue               comethod6 ?
    &CastName&_get_accDescription         comethod2 ? ; not described
    &CastName&_get_accRole                comethod2 ? ; not described
    &CastName&_get_accState               comethod2 ? ; not described
    &CastName&_get_accHelp                comethod2 ? ; not described
    &CastName&_get_accHelpTopic           comethod2 ? ; not described
    &CastName&_get_accKeyboardShortcut    comethod2 ? ; not described
    &CastName&_get_accFocus               comethod2 ? ; not described
    &CastName&_get_accSelection           comethod2 ? ; not described
    &CastName&_get_accDefaultAction       comethod2 ? ; not described
    &CastName&_accSelect                  comethod2 ? ; not described
    &CastName&_accLocation                comethod2 ? ; not described
    &CastName&_accNavigate                comethod2 ? ; not described
    &CastName&_accHitTest                 comethod2 ? ; not described
    &CastName&_accDoDefaultAction         comethod2 ? ; not described
    &CastName&_put_accName                comethod2 ? ; not described
    &CastName&_put_accValue               comethod2 ? ; not described 
ENDM

IAccessible                       STRUCT
    _vtIAccessible IAccessible
IAccessible                       ENDS

pvtIAccessible        TEXTEQU     <_vtIDispatch, accParent, \
                                 get_accChildCount, get_accChild, get_accName, get_AccValue>

WinMain                     PROTO :DWORD, :DWORD, :DWORD, :DWORD
.const
psz_NewLineW db 0ah,00h,0dh,00h
.data

DeclareGuid IID_IAccessible
vTableAccessible IAccessible <>
;Table
DeclareGuid IID_IDispatch

someplace     db 1024 dup (0)
Object        dd 0
Iacc_Object        dd 0
hText         dd 0
dummy    dd 0
hConsole   dd 0
.data?
tempVARIANT   VARIANT 100  dup(<?,?,?,?>)
aRect RECT <?,?,?,?>
.code

Start:
   invoke GetStdHandle, STD_OUTPUT_HANDLE
   mov hConsole,eax
  invoke CoInitialize, NULL
 
.repeat
  xor ecx,ecx
  lea edx,tempVARIANT
  .repeat
    push ecx
    push edx
      invoke VariantClear,edx
    pop edx
    pop ecx
    ;
    add edx,sizeof(VARIANT)
    inc ecx
  .until ecx == 10
 
  invoke GetCursorPos,addr aRect
  invoke AccessibleObjectFromPoint,aRect.left, aRect.top,addr Object,addr tempVARIANT
  .if eax == S_OK
    coinvoke Object,IDispatch,QueryInterface,pIID_IAccessible,addr Iacc_Object
    .if eax == S_OK
        coinvoke Iacc_Object,IAccessible,get_accName,VT_I4,0,0,-1,addr someplace
        .if eax == S_OK
            mov edi,dword ptr someplace
            .if edi != 0
               mov hText,edi
              invoke lstrlenW,hText
              invoke WriteConsoleW,hConsole,hText,eax,addr dummy, NULL
              invoke WriteConsoleW,hConsole,addr psz_NewLineW,2,addr dummy, NULL
            .endif
        .endif
    .endif
    coinvoke Iacc_Object,IAccessible,Release
    coinvoke Object,IAccessible,Release
  .endif
 
  invoke Sleep,100
  mov ecx,1
.until ecx == 100
 
  sortir:
    invoke CoUninitialize
    invoke ExitProcess, eax
   
end Start


hope it helps...
BasilYercin

[afk]

Hi,
I appreciate your taking the time to upload the code. Most
of it is unfamiliar to me but I'll try to figure it out over the weekend.
Again, thanks
Austin