Skip to content

feat(get_data): fallback to 'ibiblio' and add server argument #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alex-raw wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(get_data): fallback to 'ibiblio' and add server argument #41

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 35 additions & 15 deletions get_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@

import argparse
import os
import subprocess
import pickle
from subprocess import run, CalledProcessError

SERVER = "aleph.gutenberg.org::gutenberg"
SERVER_FALLBACK = "ftp.ibiblio.org::gutenberg"

if __name__ == '__main__':

Expand All @@ -22,6 +25,13 @@
"This script will download all books currently not in your\n"
"local copy of PG and get the latest version of the metadata.\n"
)

# server url
parser.add_argument(
"-s", "--server",
help="Server URL to be passed to rsync. Defaults to %s" % SERVER,
type=str)

# mirror dir
parser.add_argument(
"-m", "--mirror",
Expand Down Expand Up @@ -92,20 +102,30 @@
else:
vstring = "v"

# Pattern to match the + but not the - :
#
# + 12345 . t x t . utf 8
# - 12345 . t x t . utf8 .gzi p
# + 12345 - 0 . t x t
#---------------------------------------------
# [.-][t0][x.]t[x.] * [t8]
sp_args = ["rsync", "-am%s" % vstring,
"--include", "*/",
"--include", "[p123456789][g0123456789]%s[.-][t0][x.]t[x.]*[t8]" % args.pattern,
"--exclude", "*",
"aleph.gutenberg.org::gutenberg", args.mirror
]
subprocess.call(sp_args)
def run_rsync(server):
# Pattern to match the + but not the - :
#
# + 12345 . t x t . utf 8
# - 12345 . t x t . utf8 .gzi p
# + 12345 - 0 . t x t
#---------------------------------------------
# [.-][t0][x.]t[x.] * [t8]
sp_args=["rsync", "-am%s" % vstring,
"--include", "*/",
"--include", "[p123456789][g0123456789]%s[.-][t0][x.]t[x.]*[t8]" % args.pattern,
"--exclude", "*",
server, args.mirror
]
run(sp_args, check=True)

if args.server is None:
try:
run_rsync(SERVER)
except CalledProcessError:
print("Using fallback: %s" % SERVER_FALLBACK)
run_rsync(SERVER_FALLBACK)
else:
run_rsync(args.server)

# Get rid of duplicates
# ---------------------
Expand Down