Skip to content

Commit d8365be

Browse files
committed
Network fixes, use testnet for network tests
- testing mainnet connectivity should be done by monitoring, not by the tests in this repo - several network fixes, and refactoring to account for testnet
1 parent 8ab8654 commit d8365be

18 files changed

Lines changed: 264 additions & 74 deletions

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ USER bitmessage
3333
# Generate default config
3434
RUN pybitmessage -t
3535
# But don't include anything else
36-
RUN rm -f knownnodes.dat messages.dat debuglog singleton.lock
36+
RUN rm -f knownnodes.dat messages.dat debug.log singleton.lock
3737

3838
ENTRYPOINT ["launcher.sh"]
3939
CMD ["-d"]

buildscripts/bootstrap/start-loadbalancer.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
apt -y install curl jq ipvsadm libyajl2
44

5-
EXTIP=$(curl -s telnetmyip.com|jq -r .ip)
5+
EXTIP=$(ip route get 8.8.8.8 | \
6+
awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1); exit}')
67
if [ ! -e .env ]; then
78
THREADS=$(nproc --all)
89
PASSWORD=$(tr -dc a-zA-Z0-9 < /dev/urandom | head -c32 && echo)

src/class_objectProcessor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1026,7 +1026,7 @@ def ackDataHasAValidHeader(ackData):
10261026

10271027
magic, command, payloadLength, checksum = protocol.Header.unpack(
10281028
ackData[:protocol.Header.size])
1029-
if magic != protocol.magic:
1029+
if magic != protocol.get_magic():
10301030
logger.info('Ackdata magic bytes were wrong. Not sending ackData.')
10311031
return False
10321032
payload = ackData[protocol.Header.size:]

src/default.ini

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ blackwhitelist = black
1414
startonlogon = False
1515
showtraynotifications = True
1616
startintray = False
17+
defaultnoncetrialsperbyte = 1000
18+
defaultpayloadlengthextrabytes = 1000
1719
socksproxytype = none
1820
sockshostname = localhost
1921
socksport = 9050

src/network/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ def start(config, state): # pylint: disable=too-many-locals
4646
readKnownNodes()
4747
pool.connectToStream(1)
4848
threads_to_start = list()
49-
threads_to_start.append(BMNetworkThread)
49+
threads_to_start.extend((AddrThread, BMNetworkThread))
5050
if not config.safeGetBoolean('bootstrap', 'threads'):
51-
threads_to_start.extend([InvThread, AddrThread,
51+
threads_to_start.extend([InvThread,
5252
DownloadThread, UploadThread])
5353
for thread_class in threads_to_start:
5454
thread = thread_class(pool)

src/network/bmproto.py

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ def state_bm_header(self):
8585
self.magic, self.command, self.payloadLength, self.checksum = \
8686
protocol.Header.unpack(self.read_buf[:protocol.Header.size])
8787
self.command = self.command.rstrip(b'\x00').decode('ascii', 'replace')
88-
if self.magic != protocol.magic:
88+
if self.magic != protocol.get_magic():
8989
# skip 1 byte in order to sync
9090
self.set_state("bm_header", length=1)
9191
self.bm_proto_reset()
@@ -453,8 +453,7 @@ def _decode_addr(self):
453453

454454
def bm_command_addr(self):
455455
"""Incoming addresses, process them"""
456-
if config.safeGetBoolean('bootstrap', 'commands'):
457-
return
456+
is_bootstrap = config.safeGetBoolean('bootstrap', 'commands')
458457
# not using services
459458
for seenTime, stream, _, ip, port in self._decode_addr():
460459
ip = bytes(ip)
@@ -473,8 +472,14 @@ def bm_command_addr(self):
473472
peer = Peer(decodedIP, port)
474473

475474
with knownnodes.knownNodesLock:
476-
# isnew =
477-
knownnodes.addKnownNode(stream, peer, seenTime)
475+
if is_bootstrap:
476+
# Insert as unverified: just over 6h old so it
477+
# won't be advertised until we connect to it
478+
knownnodes.addKnownNode(
479+
stream, peer,
480+
time.time() - knownnodes.BOOTSTRAP_INSERT_AGE)
481+
else:
482+
knownnodes.addKnownNode(stream, peer, seenTime)
478483

479484
# since we don't track peers outside of knownnodes,
480485
# only spread if in knownnodes to prevent flood

src/network/connectionchooser.py

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
# pylint: disable=too-many-branches
55
import logging
66
import random
7+
import time
78

89
from six.moves import queue
910

@@ -71,11 +72,20 @@ def chooseConnection(stream):
7172
# don't connect to local IPs when using SOCKS
7273
if not protocol.checkIPAddress(encodedAddr, False):
7374
continue
75+
roll = random.random() # nosec B311
7476
if rating > 1:
7577
rating = 1
76-
try:
77-
if 0.05 / (1.0 - rating) > random.random(): # nosec B311
78+
if config.safeGetBoolean('bootstrap', 'commands'):
79+
# Bootstrap server: age-based selection with cooldown
80+
age = time.time() - peer_info.get('lastseen', 0)
81+
if age < knownnodes.BOOTSTRAP_RETRY_COOLDOWN:
82+
continue # 1h cooldown
83+
prob = float(knownnodes.BOOTSTRAP_RETRY_COOLDOWN) / age
84+
else:
85+
try:
86+
prob = 0.05 / (1.0 - rating)
87+
except ZeroDivisionError:
7888
return peer
79-
except ZeroDivisionError:
89+
if prob > roll:
8090
return peer
8191
raise ValueError

src/network/connectionpool.py

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,10 @@ def startBootstrappers(self):
238238
bootstrapper = bootstrap(connection_base)
239239
if not hostname:
240240
port = random.choice([8080, 8444]) # nosec B311
241-
hostname = 'bootstrap%s.bitmessage.org' % port
241+
if config.safeGetBoolean('bootstrap', 'testnet'):
242+
hostname = 'bootstrap%s.testnet.bitmessage.org' % port
243+
else:
244+
hostname = 'bootstrap%s.bitmessage.org' % port
242245
else:
243246
port = 8444
244247
self.addConnection(bootstrapper(hostname, port))
@@ -301,8 +304,9 @@ def loop(self): # pylint: disable=too-many-branches,too-many-statements
301304
1 for c in self.outboundConnections.values()
302305
if (c.connected and c.fullyEstablished))
303306
pending = len(self.outboundConnections) - established
304-
if established < config.safeGetInt(
305-
'bitmessagesettings', 'maxoutboundconnections'):
307+
maxOutbound = config.safeGetInt(
308+
'bitmessagesettings', 'maxoutboundconnections')
309+
if established < maxOutbound:
306310
for i in range(
307311
state.maximumNumberOfHalfOpenConnections - pending):
308312
try:
@@ -312,7 +316,9 @@ def loop(self): # pylint: disable=too-many-branches,too-many-statements
312316
continue
313317
if chosen in self.outboundConnections:
314318
continue
315-
if chosen.host in self.inboundConnections:
319+
if chosen.host in self.inboundConnections \
320+
and not config.safeGetBoolean(
321+
'bootstrap', 'dup_ip'):
316322
continue
317323
# don't connect to self
318324
if chosen in state.ownAddresses:
@@ -325,7 +331,9 @@ def loop(self): # pylint: disable=too-many-branches,too-many-statements
325331
for j in self.outboundConnections.values():
326332
if host_network_group == j.network_group:
327333
same_group = True
328-
if chosen.host == j.destination.host:
334+
if chosen.host == j.destination.host \
335+
and not config.safeGetBoolean(
336+
'bootstrap', 'commands'):
329337
knownnodes.decreaseRating(chosen)
330338
break
331339
if same_group:
@@ -398,12 +406,15 @@ def loop(self): # pylint: disable=too-many-branches,too-many-statements
398406
if i.fullyEstablished:
399407
minTx -= self.idleTimeout - 20
400408
if i.lastTx < minTx:
401-
if i.fullyEstablished:
402-
i.append_write_buf(protocol.CreatePacket('ping'))
403-
else:
409+
if not i.fullyEstablished:
404410
i.close_reason = "Timeout (%is)" % (
405411
time.time() - i.lastTx)
406412
i.set_state("close")
413+
elif config.safeGetBoolean('bootstrap', 'commands'):
414+
i.close_reason = "Bootstrap idle"
415+
i.set_state("close")
416+
else:
417+
i.append_write_buf(protocol.CreatePacket('ping'))
407418
for i in (
408419
self.connections()
409420
+ list(self.listeningSockets.values()) + list(self.udpSockets.values())

src/network/knownnodes.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@
3434

3535
knownNodesActual = False
3636

37+
# Bootstrap server address management thresholds
38+
BOOTSTRAP_INSERT_AGE = 21660
39+
"""6h + 1min: unverified addresses are inserted with this age"""
40+
41+
BOOTSTRAP_RETRY_COOLDOWN = 3600
42+
"""1 hour: don't retry connecting to a peer within this window"""
43+
3744
logger = logging.getLogger('default')
3845

3946
DEFAULT_NODES = (
@@ -78,8 +85,12 @@ def json_deserialize_knownnodes(source):
7885
info = node['info']
7986
peer = Peer(str(peer['host']), peer.get('port', 8444))
8087
knownNodes[node['stream']][peer] = info
88+
default_nodes = (
89+
TESTNET_NODES if config.safeGetBoolean('bootstrap', 'testnet')
90+
else DEFAULT_NODES
91+
)
8192
if not (knownNodesActual
82-
or info.get('self')) and peer not in DEFAULT_NODES:
93+
or info.get('self')) and peer not in default_nodes:
8394
knownNodesActual = True
8495

8596

src/network/tcp.py

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,35 @@ def __init__(self, address=None, sock=None):
9292
self.bm_proto_reset()
9393
self.set_state("bm_header", expectBytes=protocol.Header.size)
9494

95+
def _host_is_global(self):
96+
"""Is this connection's address useful for knownnodes?
97+
98+
Returns True if the peer address is a real, globally routable
99+
address that is worth sharing with the network.
100+
"""
101+
# SOCKS proxy IP: not the real peer address
102+
if protocol.checkSocksIP(self.destination.host):
103+
return False
104+
# Local/private IP: not useful to announce to the network
105+
if self.local:
106+
return False
107+
return True
108+
109+
def _getPeer(self):
110+
"""Return the peer address suitable for knownnodes.
111+
112+
For outbound connections, use destination as-is.
113+
For inbound connections, use the advertised listening port
114+
from the version message instead of the ephemeral source port.
115+
Falls back to destination if version was not yet received.
116+
"""
117+
if self.isOutbound:
118+
return self.destination
119+
try:
120+
return Peer(self.destination.host, self.peerNode.port)
121+
except AttributeError:
122+
return self.destination
123+
95124
def antiIntersectionDelay(self, initial=False):
96125
"""
97126
This is a defense against the so called intersection attacks.
@@ -162,18 +191,25 @@ def set_connection_fully_established(self):
162191
))
163192
self.antiIntersectionDelay(True)
164193
self.fullyEstablished = True
194+
peer = self._getPeer()
165195
# The connection having host suitable for knownnodes
166-
if self.isOutbound or not self.local and not state.socksIP:
167-
knownnodes.increaseRating(self.destination)
196+
if self._host_is_global():
197+
if self.isOutbound:
198+
knownnodes.increaseRating(peer)
199+
lastseen = time.time()
200+
elif config.safeGetBoolean('bootstrap', 'commands'):
201+
lastseen = time.time() - knownnodes.BOOTSTRAP_INSERT_AGE
202+
else:
203+
lastseen = time.time()
168204
knownnodes.addKnownNode(
169-
self.streams, self.destination, time.time())
205+
self.streams, peer, lastseen)
170206
dandelion_ins.maybeAddStem(self, invQueue)
171207
if not config.safeGetBoolean('bootstrap', 'addr'):
172208
self.sendAddr()
173209
if not config.safeGetBoolean('bootstrap', 'inv'):
174210
self.sendBigInv()
175211

176-
def sendAddr(self):
212+
def sendAddr(self): # pylint: disable=too-many-locals
177213
"""Send a partial list of known addresses to peer."""
178214
# We are going to share a maximum number of 1000 addrs (per overlapping
179215
# stream) with our peer. 500 from overlapping streams, 250 from the
@@ -191,11 +227,14 @@ def sendAddr(self):
191227
continue
192228
# only if more recent than 3 hours
193229
# and having positive or neutral rating
230+
is_bootstrap = config.safeGetBoolean(
231+
'bootstrap', 'commands')
194232
filtered = [
195233
(k, v) for k, v in nodes.items()
196234
if v["lastseen"] > int(time.time())
197235
- maximumAgeOfNodesThatIAdvertiseToOthers
198-
and v["rating"] >= 0 and not k.host.endswith('.onion')
236+
and (is_bootstrap or v["rating"] >= 0)
237+
and not k.host.endswith('.onion')
199238
]
200239
# sent 250 only if the remote isn't interested in it
201240
elemCount = min(
@@ -285,20 +324,21 @@ def handle_write(self):
285324

286325
def handle_close(self):
287326
"""Callback for connection being closed."""
288-
host_is_global = self.isOutbound or not self.local and not state.socksIP
327+
host_is_global = self._host_is_global()
289328
if self.fullyEstablished:
290329
UISignalQueue.put((
291330
'updateNetworkStatusTab',
292331
(self.isOutbound, False, self.destination)
293332
))
294333
if host_is_global:
295-
knownnodes.addKnownNode(
296-
self.streams, self.destination, time.time())
334+
if self.isOutbound:
335+
knownnodes.addKnownNode(
336+
self.streams, self._getPeer(), time.time())
297337
dandelion_ins.maybeRemoveStem(self)
298338
else:
299339
self.checkTimeOffsetNotification()
300-
if host_is_global:
301-
knownnodes.decreaseRating(self.destination)
340+
if host_is_global and self.isOutbound:
341+
knownnodes.decreaseRating(self._getPeer())
302342
BMProto.handle_close(self)
303343

304344

0 commit comments

Comments
 (0)