From f25f86b9c700e35704b74c0b42708794a08bd023 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 26 Sep 2025 15:35:47 +0300 Subject: [PATCH 1/3] Update funding, remove keystore --- .env.example | 5 -- src/app.py | 5 -- src/common/app_state.py | 2 - src/config/settings.py | 17 ---- src/validators/credentials.py | 54 +++++++++++ src/validators/endpoints.py | 51 +++++------ src/validators/keystore.py | 165 ---------------------------------- src/validators/schema.py | 11 --- src/validators/typings.py | 3 - src/validators/validators.py | 87 ++++++++---------- 10 files changed, 116 insertions(+), 284 deletions(-) delete mode 100644 src/validators/keystore.py diff --git a/.env.example b/.env.example index 8f50400..3744c35 100644 --- a/.env.example +++ b/.env.example @@ -6,11 +6,6 @@ RELAYER_PORT=8003 VALIDATORS_MANAGER_KEY_FILE=validators-manager-key.json VALIDATORS_MANAGER_PASSWORD_FILE=validators-manager-password.txt -# keystore -KEYSTORES_DIR=/keystores -KEYSTORES_PASSWORD_DIR=/keystores -KEYSTORES_PASSWORD_FILE=/keystores/password.txt - # choices: mainnet, hoodi, gnosis, chiado NETWORK= diff --git a/src/app.py b/src/app.py index ca6b001..79c16d8 100644 --- a/src/app.py +++ b/src/app.py @@ -11,7 +11,6 @@ from src.common.setup_logging import setup_logging from src.config import settings from src.validators.endpoints import router -from src.validators.keystore import LocalKeystore from src.validators.validators_manager import load_validators_manager_account setup_logging() @@ -27,10 +26,6 @@ async def lifespan(app_instance: FastAPI) -> AsyncIterator: # pylint:disable=un app_state.validators_manager_account = validators_manager logger.info('validators manager address: %s', validators_manager.address) - # load keystore - keystore = await LocalKeystore.load() - app_state.keystore = keystore - yield diff --git a/src/common/app_state.py b/src/common/app_state.py index ab02e83..ed1477b 100644 --- a/src/common/app_state.py +++ b/src/common/app_state.py @@ -1,9 +1,7 @@ from eth_account.signers.local import LocalAccount from src.common.typings import Singleton -from src.validators.keystore import LocalKeystore class AppState(metaclass=Singleton): validators_manager_account: LocalAccount - keystore: LocalKeystore diff --git a/src/config/settings.py b/src/config/settings.py index ebdc15f..295377a 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -1,5 +1,3 @@ -from pathlib import Path - from decouple import config from src.config.networks import NETWORKS @@ -10,19 +8,6 @@ validators_manager_key_file: str = config('VALIDATORS_MANAGER_KEY_FILE') validators_manager_password_file: str = config('VALIDATORS_MANAGER_PASSWORD_FILE') -keystores_dir = config( - 'KEYSTORES_DIR', - cast=Path, -) -keystores_password_dir = config( - 'KEYSTORES_PASSWORD_DIR', - cast=Path, -) -keystores_password_file = config( - 'KEYSTORES_PASSWORD_FILE', - cast=Path, -) - network: str = config('NETWORK') network_config = NETWORKS[network] @@ -30,8 +15,6 @@ execution_timeout: int = config('EXECUTION_TIMEOUT', cast=int, default=60) execution_retry_timeout: int = config('EXECUTION_RETRY_TIMEOUT', cast=int, default=60) -concurrency: int = config('CONCURRENCY', cast=int, default=1) - # logging LOG_PLAIN = 'plain' LOG_JSON = 'json' diff --git a/src/validators/credentials.py b/src/validators/credentials.py index 6d5f3e4..fcc2bb2 100644 --- a/src/validators/credentials.py +++ b/src/validators/credentials.py @@ -1,9 +1,13 @@ +import secrets from dataclasses import dataclass from functools import cached_property import milagro_bls_binding as bls from eth_typing import BLSPrivateKey, ChecksumAddress, HexStr from py_ecc.bls import G2ProofOfPossession +from py_ecc.optimized_bls12_381.optimized_curve import curve_order +from staking_deposit.key_handling.key_derivation.path import path_to_nodes +from staking_deposit.key_handling.key_derivation.tree import derive_child_SK from staking_deposit.settings import DEPOSIT_CLI_VERSION from sw_utils import get_v1_withdrawal_credentials, get_v2_withdrawal_credentials from sw_utils.signing import ( @@ -18,6 +22,11 @@ from src.config.networks import NETWORKS from src.validators.typings import ValidatorType +# Set path as EIP-2334 format +# https://eips.ethereum.org/EIPS/eip-2334 +PURPOSE = '12381' +COIN_TYPE = '3600' + @dataclass class Credential: @@ -75,3 +84,48 @@ def get_signed_deposit(self, amount: int) -> DepositData: signature=bls.Sign(self.private_key_bytes, signing_root), ) return signed_deposit + + +class CredentialManager: + @staticmethod + def generate_credentials( + count: int, + start_index: int, + network: str, + vault_address: ChecksumAddress, + validator_type: ValidatorType, + ) -> list[Credential]: + credentials = [] + private_key = BLSPrivateKey(secrets.randbelow(curve_order)) + for index in range(start_index, start_index + count): + credential = CredentialManager._generate_credential( + network=network, + vault=vault_address, + private_key=private_key, + index=index, + validator_type=validator_type, + ) + credentials.append(credential) + return credentials + + @staticmethod + def _generate_credential( + network: str, + vault: ChecksumAddress, + private_key: BLSPrivateKey, + index: int, + validator_type: ValidatorType, + ) -> Credential: + signing_key_path = f'm/{PURPOSE}/{COIN_TYPE}/{index}/0/0' + nodes = path_to_nodes(signing_key_path) + + for node in nodes: + private_key = BLSPrivateKey(derive_child_SK(parent_SK=private_key, index=node)) + + return Credential( + private_key=private_key, + path=signing_key_path, + network=network, + vault=vault, + validator_type=validator_type, + ) diff --git a/src/validators/endpoints.py b/src/validators/endpoints.py index 6f6dd18..4a03302 100644 --- a/src/validators/endpoints.py +++ b/src/validators/endpoints.py @@ -1,10 +1,11 @@ from fastapi import APIRouter +from sw_utils import DepositData from web3 import Web3 -from src.common.app_state import AppState from src.common.contracts import VaultContract, validators_registry_contract from src.validators import schema -from src.validators.validators import generate_validators, get_validators_for_funding +from src.validators.typings import Validator +from src.validators.validators import generate_validators from src.validators.validators_manager import ( get_validators_manager_signature_consolidation, get_validators_manager_signature_funding, @@ -15,14 +16,12 @@ router = APIRouter() -@router.post('/validators') +@router.post('/register') async def register_validators( request: schema.ValidatorsRegisterRequest, ) -> schema.ValidatorsRegisterResponse: validator_items = [] - app_state = AppState() validators = generate_validators( - keystore=app_state.keystore, vault_address=request.vault, start_index=request.validators_start_index, amounts=request.amounts, @@ -56,27 +55,26 @@ async def register_validators( @router.post('/fund') async def fund_validators( request: schema.ValidatorsFundRequest, -) -> schema.ValidatorsFundResponse: - validator_items = [] - app_state = AppState() - if not app_state.keystore: - raise ValueError('Keystore is required for funding validators') - - validators = get_validators_for_funding( - keystore=app_state.keystore, - vault_address=request.vault, - public_keys=request.public_keys, - amounts=request.amounts, - ) - - for validator in validators: - validator_items.append( - schema.ValidatorsFundResponseItem( - public_key=validator.public_key, - deposit_signature=validator.deposit_signature, - amount=validator.amount, - ) +) -> schema.ValidatorsSignatureResponse: + validators = [] + + # use empty signature and withdrawal credentials for funding + empty_signature = bytes(96) + empty_withdrawal_credentials = bytes(32) + for public_key, amount in zip(request.public_keys, request.amounts): + deposit_data = DepositData( + pubkey=Web3.to_bytes(hexstr=public_key), + withdrawal_credentials=empty_withdrawal_credentials, + amount=amount, + signature=empty_signature, + ) + validator = Validator( + public_key=public_key, + amount=amount, + deposit_signature=Web3.to_hex(empty_signature), + deposit_data_root=deposit_data.hash_tree_root, ) + validators.append(validator) vault_contact = VaultContract(request.vault) validators_manager_nonce = await vault_contact.validators_manager_nonce() @@ -86,8 +84,7 @@ async def fund_validators( validators, ) - return schema.ValidatorsFundResponse( - validators=validator_items, + return schema.ValidatorsSignatureResponse( validators_manager_signature=validators_manager_signature, ) diff --git a/src/validators/keystore.py b/src/validators/keystore.py deleted file mode 100644 index ff8156c..0000000 --- a/src/validators/keystore.py +++ /dev/null @@ -1,165 +0,0 @@ -import logging -from dataclasses import dataclass -from multiprocessing import Pool -from os import listdir -from os.path import isfile -from pathlib import Path -from typing import NewType - -import milagro_bls_binding as bls -from eth_typing import BLSPrivateKey, BLSSignature, ChecksumAddress, HexStr -from staking_deposit.key_handling.keystore import ScryptKeystore -from sw_utils.signing import get_exit_message_signing_root -from sw_utils.typings import ConsensusFork -from web3 import Web3 - -from src.config import settings -from src.validators.credentials import Credential -from src.validators.typings import BLSPrivkey, ValidatorType - -logger = logging.getLogger(__name__) - - -class KeystoreException(Exception): - ... - - -@dataclass -class KeystoreFile: - name: str - password: str - password_file: Path - - -Keys = NewType('Keys', dict[HexStr, BLSPrivkey]) - - -class LocalKeystore: - keys: Keys - - def __init__(self, keys: Keys): - self.keys = keys - - @staticmethod - async def load() -> 'LocalKeystore': - """Extracts private keys from the keys.""" - keystore_files = LocalKeystore.list_keystore_files() - logger.info('Loading keys from %s...', settings.keystores_dir) - keystores_data = [] - with Pool(processes=settings.concurrency) as pool: - # pylint: disable-next=unused-argument - def _stop_pool(*args, **kwargs): - pool.close() - - results = [ - pool.apply_async( - LocalKeystore._process_keystore_file, - (keystore_file, settings.keystores_dir), - error_callback=_stop_pool, - ) - for keystore_file in keystore_files - ] - for result in results: - result.wait() - try: - keystores_data.append(result.get()) - except KeystoreException as e: - logger.error(e) - raise RuntimeError('Failed to load keys') from e - - keys: dict[HexStr, BLSPrivkey] = {} - for pub_key, priv_key, _ in sorted(keystores_data, key=lambda x: x[2]): - keys[pub_key] = priv_key - - logger.info('Loaded %d keys', len(keys)) - return LocalKeystore(Keys(keys)) - - def __bool__(self) -> bool: - return len(self.keys) > 0 - - def __contains__(self, public_key: HexStr) -> bool: - return public_key in self.keys - - def __len__(self) -> int: - return len(self.keys) - - def get_deposit_data( - self, - public_key: HexStr, - amount: int, - vault_address: ChecksumAddress, - validator_type: ValidatorType, - ) -> dict: - private_key = self.keys[public_key] - credential = Credential( - network=settings.network, - private_key=BLSPrivateKey(Web3.to_int(private_key)), - vault=vault_address, - validator_type=validator_type, - ) - - return credential.get_deposit_datum_dict(amount) - - def get_exit_signature( - self, validator_index: int, public_key: HexStr, fork: ConsensusFork | None = None - ) -> BLSSignature: - fork = fork or settings.network_config.SHAPELLA_FORK - - private_key = self.keys[public_key] - - message = get_exit_message_signing_root( - validator_index=validator_index, - genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, - fork=fork, - ) - - return bls.Sign(private_key, message) - - @property - def public_keys(self) -> list[HexStr]: - return list(self.keys.keys()) - - @staticmethod - def list_keystore_files() -> list[KeystoreFile]: - keystores_dir = settings.keystores_dir - keystores_password_dir = settings.keystores_password_dir - keystores_password_file = settings.keystores_password_file - - res: list[KeystoreFile] = [] - for f in listdir(keystores_dir): - if not (isfile(keystores_dir / f) and f.startswith('keystore') and f.endswith('.json')): - continue - - password_file = keystores_password_dir / f.replace('.json', '.txt') - if not isfile(password_file): - password_file = keystores_password_file - - password = LocalKeystore._load_keystores_password(password_file) - res.append(KeystoreFile(name=f, password=password, password_file=password_file)) - - return res - - @staticmethod - def _process_keystore_file( - keystore_file: KeystoreFile, keystore_path: Path - ) -> tuple[HexStr, BLSPrivkey, int]: - file_name = keystore_file.name - keystores_password = keystore_file.password - file_path = keystore_path / file_name - - try: - keystore = ScryptKeystore.from_file(file_path) - except BaseException as e: - raise KeystoreException(f'Invalid keystore format in file "{file_name}"') from e - - try: - private_key = BLSPrivkey(keystore.decrypt(keystores_password)) - except BaseException as e: - raise KeystoreException(f'Invalid password for keystore "{file_name}"') from e - public_key = Web3.to_hex(bls.SkToPk(private_key)) - return public_key, private_key, int(keystore.path.split('/')[3]) - - @staticmethod - def _load_keystores_password(password_path: Path) -> str: - with open(password_path, 'r', encoding='utf-8') as f: - return f.read().strip() diff --git a/src/validators/schema.py b/src/validators/schema.py index 00d34b5..5c06a2a 100644 --- a/src/validators/schema.py +++ b/src/validators/schema.py @@ -30,17 +30,6 @@ class ValidatorsFundRequest(BaseModel): amounts: list[Gwei] -class ValidatorsFundResponseItem(BaseModel): - public_key: HexStr - deposit_signature: HexStr - amount: Gwei - - -class ValidatorsFundResponse(BaseModel): - validators: list[ValidatorsFundResponseItem] - validators_manager_signature: HexStr - - class ValidatorsWithdrawalRequest(BaseModel): vault: ChecksumAddress public_keys: list[HexStr] diff --git a/src/validators/typings.py b/src/validators/typings.py index ef79a89..32507b1 100644 --- a/src/validators/typings.py +++ b/src/validators/typings.py @@ -1,12 +1,9 @@ from dataclasses import dataclass from enum import Enum -from typing import NewType from eth_typing import BLSSignature, HexStr from web3.types import Gwei -BLSPrivkey = NewType('BLSPrivkey', bytes) - class ValidatorType(Enum): V1 = '0x01' diff --git a/src/validators/validators.py b/src/validators/validators.py index aa1ff97..376d759 100644 --- a/src/validators/validators.py +++ b/src/validators/validators.py @@ -1,70 +1,59 @@ -from eth_typing import ChecksumAddress, HexStr +import milagro_bls_binding as bls +from eth_typing import BLSSignature, ChecksumAddress +from sw_utils import ConsensusFork, get_exit_message_signing_root from web3 import Web3 from web3.types import Gwei -from src.validators.keystore import LocalKeystore +from src.config import settings +from src.validators.credentials import Credential, CredentialManager from src.validators.typings import Validator, ValidatorType def generate_validators( - keystore: LocalKeystore, vault_address: ChecksumAddress, start_index: int, amounts: list[Gwei], validator_type: ValidatorType, ) -> list[Validator]: - validators = [] + """ + `_generate_validators` generates validator keystores, but does not save keystores on disk. + todo: You should save keystores on disk to be able to exit validator manually. + """ + res = [] count = len(amounts) + credentials = CredentialManager.generate_credentials( + count=count, + start_index=0, + network=settings.network, + vault_address=vault_address, + validator_type=validator_type, + ) validator_indexes = range(start_index, start_index + count) - available_public_keys = keystore.public_keys[:count] # todo: filter non-registered - for validator_index, amount, public_key in zip( - validator_indexes, amounts, available_public_keys - ): - deposit_data = keystore.get_deposit_data( - public_key=public_key, - amount=amount, - vault_address=vault_address, - validator_type=validator_type, - ) - exit_signature = keystore.get_exit_signature( - validator_index=validator_index, public_key=Web3.to_hex(deposit_data['pubkey']) - ) - validators.append( + + for validator_index, amount, credential in zip(validator_indexes, amounts, credentials): + deposit_datum_dict = credential.get_deposit_datum_dict(amount) + exit_signature = _get_exit_signature(validator_index=validator_index, credential=credential) + res.append( Validator( - public_key=Web3.to_hex(deposit_data['pubkey']), - deposit_signature=Web3.to_hex(deposit_data['signature']), - deposit_data_root=Web3.to_hex(deposit_data['deposit_data_root']), - amount=Gwei(int(deposit_data['amount'])), + public_key=credential.public_key, + deposit_data_root=Web3.to_hex(deposit_datum_dict['deposit_data_root']), + deposit_signature=Web3.to_hex(deposit_datum_dict['signature']), + amount=amount, exit_signature=exit_signature, - validator_type=validator_type, ) ) + return res - return validators +def _get_exit_signature( + validator_index: int, credential: Credential, fork: ConsensusFork | None = None +) -> BLSSignature: + fork = fork or settings.network_config.SHAPELLA_FORK -def get_validators_for_funding( - keystore: LocalKeystore, - vault_address: ChecksumAddress, - public_keys: list[HexStr], - amounts: list[Gwei], -) -> list[Validator]: - validators = [] - for public_key, amount in zip(public_keys, amounts): - if public_key not in keystore: - raise RuntimeError(f'Public key {public_key} not found in keystores') - deposit_data = keystore.get_deposit_data( - public_key=public_key, - amount=amount, - vault_address=vault_address, - validator_type=ValidatorType.V2, - ) - validators.append( - Validator( - public_key=Web3.to_hex(deposit_data['pubkey']), - deposit_signature=Web3.to_hex(deposit_data['signature']), - amount=amount, - deposit_data_root=Web3.to_hex(deposit_data['deposit_data_root']), - ) - ) - return validators + message = get_exit_message_signing_root( + validator_index=validator_index, + genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, + fork=fork, + ) + + return bls.Sign(Web3.to_bytes(credential.private_key), message) From 31142a615d3bbe2ce9d76d92e767adaa2cb720af Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 26 Sep 2025 16:07:18 +0300 Subject: [PATCH 2/3] Use vault address for withdrawal_credentials --- src/validators/endpoints.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/validators/endpoints.py b/src/validators/endpoints.py index 4a03302..17aa8ff 100644 --- a/src/validators/endpoints.py +++ b/src/validators/endpoints.py @@ -58,13 +58,12 @@ async def fund_validators( ) -> schema.ValidatorsSignatureResponse: validators = [] - # use empty signature and withdrawal credentials for funding + # use empty signature for funding empty_signature = bytes(96) - empty_withdrawal_credentials = bytes(32) for public_key, amount in zip(request.public_keys, request.amounts): deposit_data = DepositData( pubkey=Web3.to_bytes(hexstr=public_key), - withdrawal_credentials=empty_withdrawal_credentials, + withdrawal_credentials=request.vault, amount=amount, signature=empty_signature, ) From dcc2184e54b1aaf134bdc5785e599088d82c7a57 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 29 Sep 2025 12:50:12 +0300 Subject: [PATCH 3/3] Fix withdrawal_credentials --- src/validators/endpoints.py | 6 +++--- src/validators/validators.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/validators/endpoints.py b/src/validators/endpoints.py index 17aa8ff..dcad114 100644 --- a/src/validators/endpoints.py +++ b/src/validators/endpoints.py @@ -1,5 +1,5 @@ from fastapi import APIRouter -from sw_utils import DepositData +from sw_utils import DepositData, get_v2_withdrawal_credentials from web3 import Web3 from src.common.contracts import VaultContract, validators_registry_contract @@ -63,7 +63,7 @@ async def fund_validators( for public_key, amount in zip(request.public_keys, request.amounts): deposit_data = DepositData( pubkey=Web3.to_bytes(hexstr=public_key), - withdrawal_credentials=request.vault, + withdrawal_credentials=get_v2_withdrawal_credentials(request.vault), amount=amount, signature=empty_signature, ) @@ -71,7 +71,7 @@ async def fund_validators( public_key=public_key, amount=amount, deposit_signature=Web3.to_hex(empty_signature), - deposit_data_root=deposit_data.hash_tree_root, + deposit_data_root=Web3.to_hex(deposit_data.hash_tree_root), ) validators.append(validator) diff --git a/src/validators/validators.py b/src/validators/validators.py index 376d759..6bb2aa0 100644 --- a/src/validators/validators.py +++ b/src/validators/validators.py @@ -40,6 +40,7 @@ def generate_validators( deposit_signature=Web3.to_hex(deposit_datum_dict['signature']), amount=amount, exit_signature=exit_signature, + validator_type=validator_type, ) ) return res