From 1e8e5a0c9e5d421ed55f3dd403cf20be6499d983 Mon Sep 17 00:00:00 2001 From: Lars Nt Date: Fri, 16 Oct 2020 21:34:44 +0200 Subject: [PATCH 1/3] include processor in ssh cmd line output (streamed host) --- pytranscoder/cluster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytranscoder/cluster.py b/pytranscoder/cluster.py index 2beec84..39d79eb 100644 --- a/pytranscoder/cluster.py +++ b/pytranscoder/cluster.py @@ -335,11 +335,13 @@ def go(self): ooutput = ooutput + job.media_info.ffmpeg_streams(_profile) cmd = ['-y', *oinput, '-i', self.converted_path(remote_inpath), *ooutput, self.converted_path(remote_outpath)] + processor_cmd = self.props.ffmpeg_path else: cmd = ['-i', self.converted_path(remote_inpath), *oinput, *ooutput, '-o', self.converted_path(remote_outpath)] + processor_cmd = self.props.hbcli_path - cli = [*ssh_cmd, *cmd] + cli = [*ssh_cmd, processor_cmd, *cmd] # # display useful information From c1e0509da2c0a2617ca798f26d9a15379ce82d10 Mon Sep 17 00:00:00 2001 From: Lars Nt Date: Fri, 16 Oct 2020 21:36:10 +0200 Subject: [PATCH 2/3] fix missing escapes in streaming host * quote paths in processor arguments * scp command would fail with spaces (or other special characters) in path or filename * double quotes still pose a problem but are less likely to occure * '-T' option for scp needed since strict filename checking doesn't work with quoted paths (might cause problems with pre Jan 2019 ssh/scp Versions) --- pytranscoder/cluster.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pytranscoder/cluster.py b/pytranscoder/cluster.py index 39d79eb..3f65cad 100644 --- a/pytranscoder/cluster.py +++ b/pytranscoder/cluster.py @@ -333,12 +333,12 @@ def go(self): if _profile.is_ffmpeg: if job.media_info.is_multistream() and self.configfile.automap and _profile.automap: ooutput = ooutput + job.media_info.ffmpeg_streams(_profile) - cmd = ['-y', *oinput, '-i', self.converted_path(remote_inpath), - *ooutput, self.converted_path(remote_outpath)] + cmd = ['-y', *oinput, '-i', f'"{self.converted_path(remote_inpath)}"', + *ooutput, f'"{self.converted_path(remote_outpath)}"'] processor_cmd = self.props.ffmpeg_path else: - cmd = ['-i', self.converted_path(remote_inpath), *oinput, - *ooutput, '-o', self.converted_path(remote_outpath)] + cmd = ['-i', f'"{self.converted_path(remote_inpath)}"', *oinput, + *ooutput, '-o', f'"{self.converted_path(remote_outpath)}'] processor_cmd = self.props.hbcli_path cli = [*ssh_cmd, processor_cmd, *cmd] @@ -367,7 +367,7 @@ def go(self): # trick to make scp work on the Windows side target_dir = '/' + remote_working_dir - scp = ['scp', inpath, self.props.user + '@' + self.props.ip + ':' + target_dir] + scp = ['scp', inpath, self.props.user + '@' + self.props.ip + ':"' + target_dir + '"'] self.log(' '.join(scp)) code, output = run(scp) @@ -412,7 +412,8 @@ def hb_log_callback(stats): # copy results back to local # retrieved_copy_name = os.path.join(gettempdir(), os.path.basename(remote_outpath)) - cmd = ['scp', self.props.user + '@' + self.props.ip + ':' + remote_outpath, retrieved_copy_name] + cmd = ['scp', '-T', self.props.user + '@' + self.props.ip + ':"' + remote_outpath + '"', + retrieved_copy_name] self.log(' '.join(cmd)) code, output = run(cmd) @@ -445,11 +446,11 @@ def hb_log_callback(stats): if self.props.is_windows(): remote_outpath = self.converted_path(remote_outpath) remote_inpath = self.converted_path(remote_inpath) - self.run_process([*ssh_cmd, f'"del {remote_outpath}"']) - self.run_process([*ssh_cmd, f'"del {remote_inpath}"']) + self.run_process([*ssh_cmd, 'del', f'"{remote_outpath}"']) + self.run_process([*ssh_cmd, 'del', f'"{remote_inpath}"']) else: - self.run_process([*ssh_cmd, f'"rm {remote_outpath}"']) - self.run_process([*ssh_cmd, f'"rm {remote_inpath}"']) + self.run_process([*ssh_cmd, 'rm', f'"{remote_outpath}"']) + self.run_process([*ssh_cmd, 'rm', f'"{remote_inpath}"']) finally: self.queue.task_done() From a11a9148b73977bdb8531a5283642ab8e120aaf5 Mon Sep 17 00:00:00 2001 From: Lars Nt Date: Fri, 16 Oct 2020 23:59:29 +0200 Subject: [PATCH 3/3] add streaming remote host option for copy command * add option remote_copy_cmd - string values starting with '[' are evaluated as python lists - other values are split by whitespaces into argument list * add example and description in Cluser.md --- Cluster.md | 2 ++ pytranscoder/cluster.py | 23 +++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Cluster.md b/Cluster.md index 0c34a33..a9f7771 100644 --- a/Cluster.md +++ b/Cluster.md @@ -138,6 +138,7 @@ config: user: matt # ssh login user working_dir: 'c:\temp' # working folder on remote host, required for streaming type ffmpeg: 'c:/ffmpeg/bin/ffmpeg' + remote_copy_cmd: 'scp -T' # command used to copy to and from remote machine profiles: # profiles allowed on this host - hevc_cuda - hevc_qsv @@ -175,6 +176,7 @@ config: | user | User to log into this host as via ssh. The user must be pre-authenticated to the host so that a password is not required. See https://www.ssh.com/ssh/copy-id. **[1]** | | ffmpeg | Path on the host to ffmpeg. | | working_dir | Indicates the temporary directory to use for encoding. **[2]** | +| remote_copy_cmd | Optional. Program used to copy files to and from the remote machine. Defaults to 'scp -T'. Values starting with '[' are interpreted as a python list of arguments (e.g. '["scp", "-T"]'), to allow for program path with spaces. Program must support the scp syntax (user@remote:/path/file) in both directions. Default value probably does not work with openssh versions older than January 2019. One alternative is 'rsync -e ssh'. | | profiles | The allowed profiles to use for all encodes on this host. If not provided, assumes all. A video input matching a profile that is not assigned to a particular host will be run on a host that will, if any. This is how, for example, you restrict CPU-based encodings to hosts with no hardware acceleration - or vice versa. In other words, you control how each host is used by which profiles it supports. | | path-substitutions | Optional. Applicable only to *mounted* type hosts. Use when the server media files and host mount paths are different. | | queues | Optional. You can define per-host queues to enable concurrent jobs on each host. If not given, encoding jobs will run 1 at a time. See README.md for further discussion of queues. | diff --git a/pytranscoder/cluster.py b/pytranscoder/cluster.py index 3f65cad..bd6f84e 100644 --- a/pytranscoder/cluster.py +++ b/pytranscoder/cluster.py @@ -1,6 +1,7 @@ """ Cluster support """ +import ast import datetime import os import shutil @@ -58,6 +59,22 @@ def working_dir(self): def host_type(self): return self.props['type'] + @property + def remote_copy_cmd(self) -> [str]: + if 'remote_copy_cmd' not in self.props: + return ['scp', '-T'] + + if self.props['remote_copy_cmd'][0] == '[': + try: + cmd = ast.literal_eval(self.props['remote_copy_cmd']) + except SyntaxError: + print(f'failed to parse {self.props["remote_copy_cmd"]} as python list') + sys.exit(1) + else: + cmd = self.props['remote_copy_cmd'].split() + + return cmd + def get_processor(self) -> Processor: # match first available processor (for info parsing use only) if self.ffmpeg_path: @@ -367,7 +384,9 @@ def go(self): # trick to make scp work on the Windows side target_dir = '/' + remote_working_dir - scp = ['scp', inpath, self.props.user + '@' + self.props.ip + ':"' + target_dir + '"'] + cp_cmd = self.props.remote_copy_cmd + + scp = [*cp_cmd, inpath, self.props.user + '@' + self.props.ip + ':"' + target_dir + '"'] self.log(' '.join(scp)) code, output = run(scp) @@ -412,7 +431,7 @@ def hb_log_callback(stats): # copy results back to local # retrieved_copy_name = os.path.join(gettempdir(), os.path.basename(remote_outpath)) - cmd = ['scp', '-T', self.props.user + '@' + self.props.ip + ':"' + remote_outpath + '"', + cmd = [*cp_cmd, self.props.user + '@' + self.props.ip + ':"' + remote_outpath + '"', retrieved_copy_name] self.log(' '.join(cmd))