From dfc64a618cd971993bb989a8d90a85563cfcbcbc Mon Sep 17 00:00:00 2001 From: Aryn Yklas Date: Mon, 26 Feb 2024 11:13:10 +0600 Subject: [PATCH] Better shit --- README.md | 2 +- src/block_scanner.py | 129 ++++++++++++++++------------ src/config.py | 2 +- src/functions.py | 76 ++++++++++++---- src/main.py | 195 +++++++++++++++++++++++++++--------------- src/swap_functions.py | 24 +++--- 6 files changed, 268 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index 588649f..bc5c66c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # jettonswaps -Bot for detect jetton swaps(only swaps, not liquidity) +Bot for detect jetton swaps (only swaps, not liquidity) at [DeDust.io](https://dedust.io/) ### Installing ```bash diff --git a/src/block_scanner.py b/src/block_scanner.py index 9b6e588..201383e 100644 --- a/src/block_scanner.py +++ b/src/block_scanner.py @@ -1,99 +1,114 @@ -import asyncio -import datetime -import hashlib -import logging -import time -import typing -from types import coroutine from collections import deque - -from pytoniq_core import Cell, Slice from pytoniq_core.tlb.block import ExtBlkRef - from pytoniq.liteclient import LiteClient -from pytoniq_core.tlb import Block, ValueFlow, ShardAccounts from pytoniq_core.tl import BlockIdExt -from pytoniq.liteclient.balancer import LiteBalancer +from pytoniq_core.tlb import Block +from pytoniq_core.tlb.block import BlkPrevInfo +import typing -class BlockScanner: - def __init__(self, - client: LiteClient, - block_handler: coroutine - ): - """ - :param client: LiteClient - :param block_handler: function to be called on new block - """ +class BlocksScanner: + def __init__(self, client: LiteClient, block_handler: typing.Callable[[BlockIdExt], typing.Coroutine]): self.client = client self.block_handler = block_handler - self.shards_storage = {} + self.shards_storage: dict[str, int] = {} self.blks_dequeue = deque() - self.inited = False + self.is_initialized = False - async def run(self): + async def run(self) -> None: if not self.client.inited: - raise Exception('should init client first') - master_blk = self.mc_info_to_tl_blk(await self.client.get_masterchain_info()) - if not self.inited: - shards = await self.client.get_all_shards_info(master_blk) + raise Exception("LiteClient must be initialized") + + master_blk: BlockIdExt = self._mc_info_to_tl_blk( + await self.client.get_masterchain_info() + ) + + if not self.is_initialized: + shards: list[BlockIdExt] = await self.client.get_all_shards_info(master_blk) + for shard in shards: - self.shards_storage[self.get_shard_id(shard)] = shard.seqno + self.shards_storage[self._get_shard_id(shard)] = shard.seqno self.blks_dequeue.append(shard) - self.inited = True + + self.is_initialized = True while True: self.blks_dequeue.append(master_blk) - shards = await self.client.get_all_shards_info(master_blk) + shards: list[BlockIdExt] = await self.client.get_all_shards_info(master_blk) + for shard in shards: - await self.get_not_seen_shards(shard) - self.shards_storage[self.get_shard_id(shard)] = shard.seqno + await self.parse_not_seen_shards(shard) + + self.shards_storage[self._get_shard_id(shard)] = shard.seqno while self.blks_dequeue: - await self.block_handler(self.blks_dequeue.pop()) + await self.block_handler(self.client, self.blks_dequeue.pop()) + + last_seqno: int = master_blk.seqno - last_seqno = master_blk.seqno while True: - new_master_blk = self.mc_info_to_tl_blk(await self.client.get_masterchain_info_ext()) + new_master_blk: BlockIdExt = self._mc_info_to_tl_blk( + await self.client.get_masterchain_info_ext() + ) + if new_master_blk.seqno != last_seqno: master_blk = new_master_blk break - async def get_not_seen_shards(self, shard: BlockIdExt): - if self.shards_storage.get(self.get_shard_id(shard)) == shard.seqno: - return [] - result = [] + async def parse_not_seen_shards(self, shard: BlockIdExt): + if self.shards_storage.get(self._get_shard_id(shard)) == shard.seqno: + return + self.blks_dequeue.append(shard) - full_blk = await self.client.raw_get_block_header(shard) - prev_ref = full_blk.info.prev_ref - if prev_ref.type_ == 'prev_blk_info': # only one prev block + full_blk: Block = await self.client.raw_get_block_header(shard) + prev_ref: BlkPrevInfo = full_blk.info.prev_ref + + if prev_ref.type_ == "prev_blk_info": prev: ExtBlkRef = prev_ref.prev - await self.get_not_seen_shards(BlockIdExt( - workchain=shard.workchain, seqno=prev.seqno, shard=shard.shard, - root_hash=prev.root_hash, file_hash=prev.file_hash + + await self.parse_not_seen_shards( + BlockIdExt( + workchain = shard.workchain, + seqno = prev.seqno, + shard = shard.shard, + root_hash = prev.root_hash, + file_hash = prev.file_hash ) ) + else: prev1: ExtBlkRef = prev_ref.prev1 prev2: ExtBlkRef = prev_ref.prev2 - await self.get_not_seen_shards(BlockIdExt( - workchain=shard.workchain, seqno=prev1.seqno, shard=shard.shard, - root_hash=prev1.root_hash, file_hash=prev1.file_hash + + await self.parse_not_seen_shards( + BlockIdExt( + workchain = shard.workchain, + seqno = prev1.seqno, + shard = shard.shard, + root_hash = prev1.root_hash, + file_hash = prev1.file_hash ) ) - await self.get_not_seen_shards(BlockIdExt( - workchain=shard.workchain, seqno=prev2.seqno, shard=shard.shard, - root_hash=prev2.root_hash, file_hash=prev2.file_hash + + await self.parse_not_seen_shards( + BlockIdExt( + workchain = shard.workchain, + seqno = prev2.seqno, + shard = shard.shard, + root_hash = prev2.root_hash, + file_hash = prev2.file_hash ) ) - return result @staticmethod - def mc_info_to_tl_blk(info: dict): - return BlockIdExt.from_dict(info['last']) + def _mc_info_to_tl_blk(info: dict) -> BlockIdExt: + return BlockIdExt.from_dict(info["last"]) @staticmethod - def get_shard_id(blk: BlockIdExt): - return f'{blk.workchain}:{blk.shard}' + def _get_shard_id(blk: BlockIdExt) -> str: + return "{}:{}".format( + blk.workchain, + blk.shard + ) diff --git a/src/config.py b/src/config.py index 37687fe..796c556 100644 --- a/src/config.py +++ b/src/config.py @@ -1,4 +1,4 @@ -TOKEN = 'YOUR_API_KEY_HERE' +TOKEN = 'YOUR_BOT_API_KEY_HERE' CHAT_ID = 0 # JETTON ADDRESS diff --git a/src/functions.py b/src/functions.py index f41c772..3337f73 100644 --- a/src/functions.py +++ b/src/functions.py @@ -1,35 +1,58 @@ +from pytoniq import begin_cell, Cell, Slice, LiteClient +from logger import logger + import json import hashlib -from pytoniq import begin_cell, Cell, Slice import httpx + from config import token -from logger import logger + + +CUSTOM_LP_DECIMALS: dict[str, int] = { + "jusdt": 12, + "jusdc": 12 +} + +CUSTOM_METADATA_DECIMALS: dict[str, int] = { + "jusdt": 6, + "jusdc": 6 +} + def get_json(filename): with open(f"{filename}.json", "r") as file: return json.loads(file.read()) + def put_json(content, filename): with open(f"{filename}.json", "w") as file: file.write(json.dumps(content)) -async def get_lp_price(client, symbol): - r = await client.run_get_method(address=token[symbol], method="estimate_swap_out", - stack=[ - begin_cell().store_uint(1, 4).end_cell().begin_parse(), - 1000000000 - ]) - if symbol in ["jusdt", "jusdc"]: - return r[1] / 1e12 + +async def get_lp_price(client: LiteClient, symbol: str): + r = await client.run_get_method( + address = token[symbol], + method = "estimate_swap_out", + stack = [ + begin_cell().store_uint(1, 4).end_cell().begin_parse(), + 1000000000 + ] + ) + + if symbol in CUSTOM_LP_DECIMALS: + return r[1] / 10 ** CUSTOM_LP_DECIMALS[symbol] + return r[1] / 1e9 def get_hash(string: str) -> int: return int.from_bytes(hashlib.sha256(string.encode()).digest(), 'big') + def get_str_attr_value(meta: dict, attr_name: str): hash_ = get_hash(attr_name) value: Slice = meta.get(hash_) + if value is not None: return value.load_snake_string().replace('\x00', '') @@ -37,18 +60,21 @@ def get_str_attr_value(meta: dict, attr_name: str): def get_bytes_attr_value(meta: dict, attr_name: str) -> bytes: hash_ = get_hash(attr_name) value: Slice = meta.get(hash_) + if value is not None: return value.load_snake_bytes() def process_metadata(cell: Cell): cs = cell.begin_parse() + if not len(cell.refs): # some metadata cells do not have b'\x01' prefix - url = cs.load_snake_string().replace('\x01', '') - return url + return cs.load_snake_string().replace('\x01', '') + else: cs.load_uint(8) metadata = cs.load_dict(key_length=256) + if metadata is None: return {} @@ -69,27 +95,43 @@ def process_metadata(cell: Cell): return result -async def get_jetton_content_by_jetton_wallet(client, jetton_wallet): + +async def get_jetton_content_by_jetton_wallet(client: LiteClient, jetton_wallet): result = {} - stack = await client.run_get_method(address=jetton_wallet, method="get_wallet_data", stack=[]) + stack = await client.run_get_method( + address = jetton_wallet, + method = "get_wallet_data", + stack = [] + ) + result["jetton_address"] = stack[2].load_address().to_str(1, 1, 1) - stack = await client.run_get_method(address=result["jetton_address"], method="get_jetton_data", stack=[]) + stack = await client.run_get_method( + address = result["jetton_address"], + method = "get_jetton_data", + stack = [] + ) + metadata = process_metadata(stack[3]) + if isinstance(metadata, str): async with httpx.AsyncClient() as async_client: if metadata[0:4] == "ipfs": r = await async_client.get(f"https://ipfs.io/ipfs/{metadata.replace('ipfs://', '')}") else: r = await async_client.get(metadata) - r = r.json() + + r: dict = r.json() + result["decimals"] = r.get("decimals", 9) result["symbol"] = r.get("symbol", "None") + elif not metadata["symbol"]: async with httpx.AsyncClient() as async_client: r = await async_client.get(metadata["uri"]) - r = r.json() + r: dict = r.json() + result["decimals"] = r.get("decimals", 9) result["symbol"] = r.get("symbol", "None") diff --git a/src/main.py b/src/main.py index f3a83f3..bda739c 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,5 @@ import asyncio -from pytoniq import BlockIdExt, Address, LiteBalancer +from pytoniq import BlockIdExt, Address, LiteBalancer, Transaction from block_scanner import BlockScanner from aiogram import Bot from logger import logger @@ -7,159 +7,214 @@ from functions import get_json, put_json, get_jetton_content_by_jetton_wallet from config import TOKEN, CHAT_ID, addresses, flipped_token -bot = Bot(token=TOKEN, parse_mode="HTML") +bot = Bot( + token = TOKEN, + parse_mode = "HTML" +) async def handle_block(block: BlockIdExt): if block.workchain == -1: # skip masterchain blocks return + try: - transactions = await client.raw_get_block_transactions_ext(block) + transactions: list[Transaction] = await client.raw_get_block_transactions_ext(block) - logger.info(f"New block: {block}") - transactions_hashes = {} # dict[transaction_hash: transaction] - msg_hashes = {} # dict[msg_hash: transaction_hash] - traces = {} # dict[transaction_child_hash: transaction_parent_hash] + transactions_hashes: dict[bytes, Transaction] = {} + msg_hashes: dict[bytes, bytes] = {} + traces: dict[bytes, bytes] = {} - transactions = sorted(transactions, key=lambda x: x.lt) + transactions = sorted( + transactions, + key = lambda transaction: transaction.lt + ) for transaction in transactions: - tr_hash = transaction.cell.hash - in_msg_hash = transaction.in_msg.serialize().hash + tr_hash: bytes = transaction.cell.hash + in_msg_hash: bytes = transaction.in_msg.serialize().hash + transactions_hashes[tr_hash] = transaction + if in_msg_hash in msg_hashes: traces[tr_hash] = msg_hashes[in_msg_hash] + if transaction.out_msgs: for msg in transaction.out_msgs: msg_hashes[msg.serialize().hash] = tr_hash - else: # end of the trace - result = [] - pretty_result = [] - - build_trace(tr_hash, transactions_hashes, traces, result) - result = result[::-1] - for tr in result: - if not tr.in_msg.is_internal: + + else: + results: list[Transaction] = [] + pretty_result: list[dict] = [] + + build_trace(tr_hash, transactions_hashes, traces, results) + + results = results[::-1] + + for transaction in results: + if not transaction.in_msg.is_internal: continue - if len(tr.in_msg.body.bits) < 32: - pretty_result.append({"type": "TonTransfer", - "amount": tr.in_msg.info.value_coins, - "destination": tr.in_msg.info.dest.to_str(1, 1, 1), - "from": tr.in_msg.info.src.to_str(1, 1, 1), - "lt": tr.lt}) + + if len(transaction.in_msg.body.bits) < 32: + pretty_result.append(dict( + type_ = "TonTransfer", + amount = transaction.in_msg.info.value_coins, + destination = transaction.in_msg.info.dest.to_str( + is_user_friendly = False + ), + from_ = transaction.in_msg.info.src.to_str( + is_user_friendly = False + ), + lt = transaction.lt + )) + continue - body_slice = tr.in_msg.body.begin_parse() + body_slice = transaction.in_msg.body.begin_parse() op_code = hex(body_slice.load_uint(32)) if op_code == "0xf8a7ea5": body_slice.load_uint(64) # skip query_id - jetton = await get_jetton_content_by_jetton_wallet(client, "0:" + tr.account_addr_hex) - pretty_result.append({"type": "JettonTransfer", - "amount": tr.in_msg.info.value_coins, - "jetton_amount": body_slice.load_coins(), - "destination": body_slice.load_address().to_str(1, 1, 1), - "from": tr.in_msg.info.src.to_str(1, 1, 1), - "op-code": op_code, - "lt": transaction.lt, - "jetton": jetton}) + jetton = await get_jetton_content_by_jetton_wallet( + client, + "0:" + transaction.account_addr_hex + ) + + pretty_result.append({ + "type": "JettonTransfer", + "amount": transaction.in_msg.info.value_coins, + "jetton_amount": body_slice.load_coins(), + "destination": body_slice.load_address().to_str(1, 1, 1), + "from": transaction.in_msg.info.src.to_str(1, 1, 1), + "op-code": op_code, + "lt": transaction.lt, + "jetton": jetton + }) elif op_code == "0x72aca8aa": body_slice.load_uint(64) # skip query_id body_slice.load_ref() # skip proof asset = {} + if body_slice.load_uint(4) == 1: # is jetton - asset = {"address": Address((body_slice.load_uint(8), body_slice.load_bytes(32)))} - else: - asset = {"address": Address((0, bytes(32)))} + asset = {"address": Address(( + body_slice.load_uint(8), + body_slice.load_bytes(32) + ))} - pretty_result.append({"type": "Routing", - "asset": asset, - "amount": tr.in_msg.info.value_coins, - "destination": tr.in_msg.info.dest.to_str(1, 1, 1), - "from": tr.in_msg.info.src.to_str(1, 1, 1), - "op-code": op_code, - "lt": tr.lt}) + else: + asset = {"address": Address(( + 0, + bytes(32) + ))} + + pretty_result.append({ + "type": "Routing", + "asset": asset, + "amount": transaction.in_msg.info.value_coins, + "destination": transaction.in_msg.info.dest.to_str(1, 1, 1), + "from": transaction.in_msg.info.src.to_str(1, 1, 1), + "op-code": op_code, + "lt": transaction.lt + }) elif op_code == "0x61ee542d": body_slice.load_uint(64) # skip query_id body_slice.load_ref() # skip proof - pretty_result.append({"type": "PayIn", - "amount": tr.in_msg.info.value_coins, - "pay_amount": body_slice.load_coins(), - "pay_sender": body_slice.load_address().to_str(1, 1, 1), - "destination": tr.in_msg.info.dest.to_str(1, 1, 1), - "from": tr.in_msg.info.src.to_str(1, 1, 1), - "op-code": op_code, - "lt": tr.lt}) + pretty_result.append({ + "type": "PayIn", + "amount": transaction.in_msg.info.value_coins, + "pay_amount": body_slice.load_coins(), + "pay_sender": body_slice.load_address().to_str(1, 1, 1), + "destination": transaction.in_msg.info.dest.to_str(1, 1, 1), + "from": transaction.in_msg.info.src.to_str(1, 1, 1), + "op-code": op_code, + "lt": transaction.lt + }) elif op_code == "0xad4eb6f5": body_slice.load_uint(64) # skip query_id body_slice.load_ref() # skip proof - pretty_result.append({"type": "PayOut", - "amount": tr.in_msg.info.value_coins, - "pay_amount": body_slice.load_coins(), - "pay_sender": body_slice.load_address().to_str(1, 1, 1), - "destination": tr.in_msg.info.dest.to_str(1, 1, 1), - "from": tr.in_msg.info.src.to_str(1, 1, 1), - "op-code": op_code, - "lt": tr.lt}) + pretty_result.append({ + "type": "PayOut", + "amount": transaction.in_msg.info.value_coins, + "pay_amount": body_slice.load_coins(), + "pay_sender": body_slice.load_address().to_str(1, 1, 1), + "destination": transaction.in_msg.info.dest.to_str(1, 1, 1), + "from": transaction.in_msg.info.src.to_str(1, 1, 1), + "op-code": op_code, + "lt": transaction.lt + }) elif op_code not in ["0x178d4519", "0x7362d09c"]: - pretty_result.append({"type": "SmartContractExec", - "amount": tr.in_msg.info.value_coins, - "destination": tr.in_msg.info.dest.to_str(1, 1, 1), - "from": tr.in_msg.info.src.to_str(1, 1, 1), - "op-code": op_code, - "lt": tr.lt}) + pretty_result.append({ + "type": "SmartContractExec", + "amount": transaction.in_msg.info.value_coins, + "destination": transaction.in_msg.info.dest.to_str(1, 1, 1), + "from": transaction.in_msg.info.src.to_str(1, 1, 1), + "op-code": op_code, + "lt": transaction.lt + }) for pretty_data in pretty_result: jetton_address = None + if pretty_data["type"] == "JettonTransfer": jetton_address = pretty_data["jetton"]["jetton_address"] - if (pretty_data["destination"] in flipped_token or pretty_data["from"] in flipped_token) or (jetton_address in addresses): + if pretty_data["destination"] in flipped_token or pretty_data["from"] in flipped_token or jetton_address in addresses: raw_results = init_raw_result(pretty_result) old_results = get_json("old_results") + for raw_result in raw_results: if raw_result in old_results: continue + old_results.append(raw_result) + while len(old_results) > 10: old_results.pop(0) + put_json(old_results, "old_results") uf_result = await raw_to_userfriendly(client, raw_result, transaction.cell.hash.hex()) + await bot.send_message(CHAT_ID, uf_result, disable_web_page_preview=True) break + except Exception as e: logger.error(f"Handle_block error: {e}") + client = LiteBalancer.from_mainnet_config(1) + def build_trace(tr_hash: str, transactions_hashes: dict, traces: dict, result: list): result.append(transactions_hashes.get(tr_hash)) + if tr_hash in traces: # we have this trace build_trace(traces[tr_hash], transactions_hashes, traces, result) + return + async def main(): logger.info("Start") + while True: try: await client.start_up() await BlockScanner(client=client, block_handler=handle_block).run() + except asyncio.TimeoutError: await client.close_all() await asyncio.sleep(3) continue + if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.create_task(main()) - loop.run_forever() + asyncio.run(main()) diff --git a/src/swap_functions.py b/src/swap_functions.py index 02a967e..25c78c7 100644 --- a/src/swap_functions.py +++ b/src/swap_functions.py @@ -1,13 +1,19 @@ import copy -from pytoniq import Address +from pytoniq import LiteClient, Address from logger import logger from functions import get_lp_price, get_json, put_json from config import token, flipped_token, known_wallets -symbols = {"sold": "🔴", "bought": "🟢", - "withdraw": "💀", "deposit": "👛"} -async def raw_to_userfriendly(client, raw_result, tx): +symbols = { + "sold": "🔴", + "bought": "🟢", + "withdraw": "💀", + "deposit": "👛" +} + + +async def raw_to_userfriendly(client: LiteClient, raw_result: dict, tx: str) -> str: result = None price = None sell_or_buy = None @@ -245,16 +251,6 @@ def init_raw_result(data): for i in raw_results: if all([bool(i["station"]), bool(i["amounts"][0]), bool(i["amounts"][1]), bool(i["who"]), bool(i["symbols"][0]), bool(i["symbols"][1])]): - # if not i["is_multi-hop"]: - # if i["station"] == "bought": - # i["amounts"][1] -= 0.2 - # if i["station"] == "sold": - # i["amounts"][1] -= 0.105 - # if i["station"] == "deposit": - # i["amounts"][1] -= 0.15 - # if i["station"] == "withdraw": - # i["amounts"][1] -= 0.225 - if i["symbols"][0] == i["symbols"][1]: continue