Rc4 help

[ecube]

The rc4 alg below works fine but for what i'm using it for I can't use  rc4keytable directly, I need it in a register. Lea eax,rc4keytable for example, only eax isn't free. Thanks, i've tried everything I can think of.

Code Select Expand


.586 
.model flat, stdcall
option casemap:none
;extern Resource:byte

include \masm32\include\windows.inc

include \masm32\include\kernel32.inc
includelib \masm32\lib\kernel32.lib

include \masm32\include\user32.inc
includelib \masm32\lib\user32.lib

include \masm32\include\masm32.inc
includelib \masm32\lib\masm32.lib

include \masm32\macros\macros.asm

_rc4_crypt proto :DWORD,:DWORD
_rc4_setkey proto :DWORD, :DWORD

.data
rc4keytable byte 256 dup (0)
Testdata db "Success",0
MyKey    db "abc123",0

.code

start:

invoke lstrlen,addr MyKey
push eax
push offset MyKey
call _rc4_setkey

invoke lstrlen,addr Testdata
push eax
push offset Testdata
call _rc4_crypt
invoke MessageBox,0,addr Testdata,addr Testdata,MB_OK


invoke lstrlen,addr MyKey
push eax
push offset MyKey
call _rc4_setkey

invoke lstrlen,addr Testdata
push eax
push offset Testdata
call _rc4_crypt
invoke MessageBox,0,addr Testdata,addr Testdata,MB_OK


invoke CloseHandle,eax
invoke ExitProcess,0

_rc4_setkey proc ptrPass:DWORD, lPass:DWORD
pushad

mov eax, 0FFFEFDFCh
mov ecx, 256/4
_init_rc4keytable:
mov dword ptr [rc4keytable+4*ecx-4], eax
sub eax, 04040404h
dec ecx
jnz _init_rc4keytable

xor eax, eax
mov edi, ptrPass

_key_return:
xor ebx, ebx
mov esi , lPass
jmp _new_key

_key_loop:
inc bl
dec esi
jz _key_return

_new_key:
mov dl, byte ptr [rc4keytable+ecx]
add al, byte ptr [edi+ebx]
add al, dl
mov dh, byte ptr [rc4keytable+eax]
mov byte ptr [rc4keytable+ecx], dh
mov byte ptr [rc4keytable+eax], dl
inc cl
jnz _key_loop

popad
ret
_rc4_setkey endp

_rc4_crypt proc ptrData:DWORD, lData:DWORD
pushad
mov edi, lData
mov esi, ptrData
test edi, edi
jz _rc4_enc_exit

xor eax, eax
xor ebx, ebx
xor ecx, ecx
xor edx, edx

_rc4_enc_loop:
inc bl
mov dl, byte ptr [rc4keytable+ebx]
add al, dl
mov cl, byte ptr [rc4keytable+eax]
mov byte ptr [rc4keytable+ebx], cl
mov byte ptr [rc4keytable+eax], dl
add cl, dl
mov cl, byte ptr [rc4keytable+ecx]
xor byte ptr [esi], cl
inc esi
dec edi
jnz _rc4_enc_loop

xor eax, eax
mov edi, offset rc4keytable
mov ecx, 256/4
cld
rep stosd

_rc4_enc_exit:
popad
ret
_rc4_crypt endp





end start


[Tedd]

Do a bit of re-arranging, and store your 'loop-counter' on the stack - since you only use it at the end of each loop - which will free up that register.

An alternative, since you're not using any local (stack-based) variables, is to turn off the stack-frame, reference the parameters in terms of esp, and then you've got ebp as a spare register.


I'd avoid pushad/popad - it's slooowwwww. The usual convention is to save esi,edi, and ebx (and ebp if you use it directly; the stack-frame setup saves it when it uses it.)