The rc4 alg below works fine but for what i'm using it for I can't use rc4keytable directly, I need it in a register. Lea eax,rc4keytable for example, only eax isn't free. Thanks, i've tried everything I can think of.
.586
.model flat, stdcall
option casemap:none
;extern Resource:byte
include \masm32\include\windows.inc
include \masm32\include\kernel32.inc
includelib \masm32\lib\kernel32.lib
include \masm32\include\user32.inc
includelib \masm32\lib\user32.lib
include \masm32\include\masm32.inc
includelib \masm32\lib\masm32.lib
include \masm32\macros\macros.asm
_rc4_crypt proto :DWORD,:DWORD
_rc4_setkey proto :DWORD, :DWORD
.data
rc4keytable byte 256 dup (0)
Testdata db "Success",0
MyKey db "abc123",0
.code
start:
invoke lstrlen,addr MyKey
push eax
push offset MyKey
call _rc4_setkey
invoke lstrlen,addr Testdata
push eax
push offset Testdata
call _rc4_crypt
invoke MessageBox,0,addr Testdata,addr Testdata,MB_OK
invoke lstrlen,addr MyKey
push eax
push offset MyKey
call _rc4_setkey
invoke lstrlen,addr Testdata
push eax
push offset Testdata
call _rc4_crypt
invoke MessageBox,0,addr Testdata,addr Testdata,MB_OK
invoke CloseHandle,eax
invoke ExitProcess,0
_rc4_setkey proc ptrPass:DWORD, lPass:DWORD
pushad
mov eax, 0FFFEFDFCh
mov ecx, 256/4
_init_rc4keytable:
mov dword ptr [rc4keytable+4*ecx-4], eax
sub eax, 04040404h
dec ecx
jnz _init_rc4keytable
xor eax, eax
mov edi, ptrPass
_key_return:
xor ebx, ebx
mov esi , lPass
jmp _new_key
_key_loop:
inc bl
dec esi
jz _key_return
_new_key:
mov dl, byte ptr [rc4keytable+ecx]
add al, byte ptr [edi+ebx]
add al, dl
mov dh, byte ptr [rc4keytable+eax]
mov byte ptr [rc4keytable+ecx], dh
mov byte ptr [rc4keytable+eax], dl
inc cl
jnz _key_loop
popad
ret
_rc4_setkey endp
_rc4_crypt proc ptrData:DWORD, lData:DWORD
pushad
mov edi, lData
mov esi, ptrData
test edi, edi
jz _rc4_enc_exit
xor eax, eax
xor ebx, ebx
xor ecx, ecx
xor edx, edx
_rc4_enc_loop:
inc bl
mov dl, byte ptr [rc4keytable+ebx]
add al, dl
mov cl, byte ptr [rc4keytable+eax]
mov byte ptr [rc4keytable+ebx], cl
mov byte ptr [rc4keytable+eax], dl
add cl, dl
mov cl, byte ptr [rc4keytable+ecx]
xor byte ptr [esi], cl
inc esi
dec edi
jnz _rc4_enc_loop
xor eax, eax
mov edi, offset rc4keytable
mov ecx, 256/4
cld
rep stosd
_rc4_enc_exit:
popad
ret
_rc4_crypt endp
end start
Do a bit of re-arranging, and store your 'loop-counter' on the stack - since you only use it at the end of each loop - which will free up that register.
An alternative, since you're not using any local (stack-based) variables, is to turn off the stack-frame, reference the parameters in terms of esp, and then you've got ebp as a spare register.
I'd avoid pushad/popad - it's slooowwwww. The usual convention is to save esi,edi, and ebx (and ebp if you use it directly; the stack-frame setup saves it when it uses it.)