efas download is prone to timeout

[jacopo-exact]

This happens also after a successful run, i.e., when it should already have everything in cache.

Traceback (most recent call last):
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 1318, in context
    yield client
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/ogs_riverger/efas/download_tools.py", line 817, in _get_remote_server_available_files
    for p, p_stat in await client.list(EFAS_FTP_DIR)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/common.py", line 130, in _to_list
    async for item in self:
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 852, in __anext__
    cls.stream = await cls._new_stream(path)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 836, in _new_stream
    return await self.get_stream(command, "1xx")
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 1231, in get_stream
    reader, writer = await self.get_passive_connection(conn_type)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 1210, in get_passive_connection
    reader, writer = await self._open_connection(ip, port)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/streams.py", line 48, in open_connection
    transport, _ = await loop.create_connection(
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/base_events.py", line 1085, in create_connection
    raise exceptions[0]
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/base_events.py", line 1069, in create_connection
    sock = await self._connect_sock(
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/base_events.py", line 973, in _connect_sock
    await self.sock_connect(sock, address)
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/selector_events.py", line 634, in sock_connect
    return await fut
           ^^^^^^^^^
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/selector_events.py", line 674, in _sock_connect_cb
    raise OSError(err, f'Connect call failed {address}')
TimeoutError: [Errno 110] Connect call failed ('136.156.254.242', 1024)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/leonardo/home/usera07ogs/a07ogs04/venv-airflow/lib/python3.11/site-packages/airflow/models/taskinstance.py", line 768, in _execute_task
    result = _execute_callable(context=context, **execute_callable_kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/home/usera07ogs/a07ogs04/venv-airflow/lib/python3.11/site-packages/airflow/models/taskinstance.py", line 734, in _execute_callable
    return ExecutionCallableRunner(
           ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/home/usera07ogs/a07ogs04/venv-airflow/lib/python3.11/site-packages/airflow/utils/operator_helpers.py", line 252, in run
    return self.func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/home/usera07ogs/a07ogs04/venv-airflow/lib/python3.11/site-packages/airflow/models/baseoperator.py", line 424, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/home/usera07ogs/a07ogs04/venv-airflow/lib/python3.11/site-packages/pydantic_provider/__init__.py", line 61, in execute
    result = self.python_callable(*_args, **_kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/mer/tasks/preproc_tasks.py", line 274, in download_river_data_task
    downloaded_files = asyncio.run(
                       ^^^^^^^^^^^^
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/runners.py", line 190, in run
    return runner.run(main)
           ^^^^^^^^^^^^^^^^
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/runners.py", line 118, in run
    return self._loop.run_until_complete(task)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/asyncio/base_events.py", line 653, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/ogs_riverger/efas/download_tools.py", line 975, in download_efas_operational_data
    available_files = await self._get_remote_server_available_files()
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/ogs_riverger/efas/download_tools.py", line 809, in _get_remote_server_available_files
    async with aioftp.Client.context(
  File "/leonardo/prod/spack/5.2/install/0.21/linux-rhel8-icelake/gcc-8.5.0/python-3.11.6-i5k3c6ggftqkzgqyymfbkynpgm2lgjtd/lib/python3.11/contextlib.py", line 222, in __aexit__
    await self.gen.athrow(typ, value, traceback)
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 1320, in context
    await client.quit()
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 1154, in quit
    await self.command("QUIT", "2xx")
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 281, in command
    self.check_codes(expected_codes, code, info)
  File "/leonardo_work/PNRR_MER/venv-mer/lib/python3.11/site-packages/aioftp/client.py", line 231, in check_codes
    raise errors.StatusCodeError(expected_codes, received_code, info)
aioftp.errors.StatusCodeError: Waiting for ('2xx',) but got 425 [" Can't open data connection (Accept timed out)"]

[spiani]

This also happens even when everything is cached, because it still checks the remote server for any new files. For now, I’ve set a shorter timeout (30 seconds) and enabled automatic retries (up to 10 attempts) (in #11)