Skip to content

Commit e30f40c

Browse files
PostgresNodeLogReader is verified and updated (#391)
New code processes completed log lines only.
1 parent 5c561fd commit e30f40c

1 file changed

Lines changed: 40 additions & 13 deletions

File tree

src/node.py

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2370,9 +2370,11 @@ def _delim_sql_ident(name: str) -> str:
23702370
class PostgresNodeLogReader:
23712371
class LogInfo:
23722372
position: int
2373+
tail: bytes
23732374

23742375
def __init__(self, position: int):
23752376
self.position = position
2377+
self.tail = b''
23762378

23772379
# --------------------------------------------------------------------
23782380
class LogDataBlock:
@@ -2448,38 +2450,63 @@ def read(self) -> typing.List[LogDataBlock]:
24482450
assert type(file_name) is str
24492451
assert type(cur_log_info) is __class__.LogInfo
24502452

2451-
read_pos = 0
2452-
2453-
if file_name in self._logs.keys():
2453+
if file_name not in self._logs.keys():
2454+
read_pos = 0
2455+
file_content_b = b''
2456+
else:
24542457
prev_log_info = self._logs[file_name]
24552458
assert type(prev_log_info) is __class__.LogInfo
24562459
read_pos = prev_log_info.position # the previous size
2460+
file_content_b = prev_log_info.tail
24572461

2458-
file_content_b = self._node.os_ops.read_binary(file_name, read_pos)
2462+
prev_data_sz = len(file_content_b)
2463+
assert prev_data_sz <= read_pos
2464+
2465+
file_content_b += self._node.os_ops.read_binary(file_name, read_pos)
24592466
assert type(file_content_b) is bytes
24602467

2468+
assert prev_data_sz <= len(file_content_b)
2469+
24612470
#
2462-
# A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol.
2471+
# We will process completed lines only
24632472
#
2464-
file_content_s = file_content_b.decode()
2465-
assert type(file_content_s) is str
2473+
completed_data_size = file_content_b.rfind(b"\n") + 1
24662474

2467-
next_read_pos = read_pos + len(file_content_b)
2475+
assert completed_data_size >= 0
2476+
assert completed_data_size <= len(file_content_b)
24682477

2469-
# It is a research/paranoja check.
2470-
# When we will process partial UTF-8 symbol, it must be adjusted.
2471-
assert cur_log_info.position <= next_read_pos
2478+
completed_data = file_content_b[:completed_data_size]
2479+
assert type(completed_data) is bytes
2480+
assert len(completed_data) == completed_data_size
24722481

2473-
cur_log_info.position = next_read_pos
2482+
new_tail = file_content_b[completed_data_size:]
2483+
assert type(new_tail) is bytes
2484+
assert len(new_tail) == len(file_content_b) - completed_data_size
2485+
2486+
completed_data_s = completed_data.decode()
2487+
assert type(completed_data_s) is str
2488+
2489+
next_read_pos = read_pos - prev_data_sz + len(file_content_b)
2490+
2491+
assert read_pos <= next_read_pos
2492+
2493+
# It is a FINAL paranoja check.
2494+
# [2026-07-10] Verified
2495+
assert cur_log_info.position <= next_read_pos
24742496

24752497
block = __class__.LogDataBlock(
24762498
file_name,
24772499
read_pos,
2478-
file_content_s
2500+
completed_data_s,
24792501
)
24802502

24812503
result.append(block)
24822504

2505+
# Save information to next iteration
2506+
cur_log_info.position = next_read_pos
2507+
cur_log_info.tail = new_tail
2508+
continue
2509+
24832510
# A new check point
24842511
self._logs = cur_logs
24852512

0 commit comments

Comments
 (0)