FPU problem

[minor28]

I want to see if the value of a hex string not is greater than 0FFFFFFFFh. To convert the string to hex I used shlwapi function StrToIntEX. But it will not work as I want. There is another function StrToIntEx64 but it is a newer version of shlwapi.

Then I write my own StrToIntEx64.

Code Select Expand


StrToIntEx64 proc pText:dword,dwFlags:dword,retval:dword
LOCAL val:dword
LOCAL res:qword
LOCAL k:dword

mov eax,ebp
.while eax>esp
dec eax
mov byte ptr [eax],0
.endw

.if dwFlags==STIF_SUPPORT_HEX
mov k,16
.elseif dwFlags==STIF_DEFAULT
mov k,10
.endif

finit
push 0
push 0
fild qword ptr [esp]
add esp,8

mov edi,pText
invoke lstrlen,edi
push eax
add edi,eax
dec edi
movzx eax,byte ptr [edi]
dec edi
.if eax>=31h && eax<=39h
sub eax,30h
.elseif eax>=41h && eax<=46h
sub eax,37h
.endif
push 0
push eax
fild qword ptr [esp]
add esp,8
faddp st(1),st(0)

pop eax
xor ecx,ecx
inc ecx
.while ecx<=eax
push ecx
push eax
movzx eax,byte ptr [edi]
dec edi
.if eax>=31h && eax<=39h
sub eax,30h
.elseif eax>=41h && eax<=46h
sub eax,37h
.endif
mov val,eax
pop eax
pop ecx
push ecx
push eax
mov eax,1
.while ecx>0
xor edx,edx
mul k
dec ecx
.endw
mul val
push 0
push eax
fild qword ptr [esp]
add esp,8
faddp st(1),st(0)
pop eax
pop ecx
inc ecx
.endw
fistp res
lea edi,res
mov eax,dword ptr [edi]
mov retval,eax
mov eax,dword ptr [edi+4]

ret

StrToIntEx64 endp


From the string "abccddeef" (longer than a dword) I want the function to return the dword 0bccddeefh in retval and the "a" in eax. The problem is the fistp res operation which gives res=00000000bccddeef.

Any idea how to solve this or a better way to know if the string is an overflow dword?

[dedndave]

if it has more than 8 hex characters, it is too large for a dword

[minor28]

I found the problem

Code Select Expand


mov eax,1
.while ecx>0
xor edx,edx
mul k
dec ecx
.endw

eax becomes 0 for the "a"

Edit:
This solved the problem

Code Select Expand


push 1
fild dword ptr [esp]
add esp,4
.while ecx>0
push k
fimul dword ptr [esp]
add esp,4
dec ecx
.endw
push val
fimul dword ptr [esp]
add esp,4
faddp st(1),st(0)
pop eax
pop ecx
inc ecx
.endw
fistp res