From 222cc8823b4532cebbda21aae48e0ea92c2bd38e Mon Sep 17 00:00:00 2001 From: Matthijs van Niekerk Date: Wed, 24 Apr 2024 15:26:22 +0200 Subject: [PATCH 1/5] INFRA-415 create recon proprocess docker file and parameterize output dir for preprocess script --- panel/Dockerfile | 4 ++++ panel/TransformData.py | 38 ++++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 panel/Dockerfile diff --git a/panel/Dockerfile b/panel/Dockerfile new file mode 100644 index 000000000..643f522ab --- /dev/null +++ b/panel/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /opt/app +COPY . . +ENTRYPOINT ["python", "TransformData.py"] \ No newline at end of file diff --git a/panel/TransformData.py b/panel/TransformData.py index b6b61600e..ff012e782 100644 --- a/panel/TransformData.py +++ b/panel/TransformData.py @@ -67,14 +67,15 @@ def getPurityPloidy(purple_purity): res['purity'] = float(arr[header.index('purity')]) res['ploidy'] = float(arr[header.index('ploidy')]) return res -def transformRatioFile(cobalt, sampleId,geneLocation): + + +def transformRatioFile(cobalt, sampleId,geneLocation, output_dir='transformed_data'): ## xx minTumorGCRatio=10 zeroGCoffset = 0.2 - - ft=open('transformed_data/'+sampleId+'.transformed.cnr','w') + ft=open(output_dir + '/' + sampleId + '.transformed.cnr', 'w') first = True header=[] @@ -127,11 +128,11 @@ def transformRatioFile(cobalt, sampleId,geneLocation): ft.write(lines+tsvSplit+start+tsvSplit+end+tsvSplit+str(logR)+tsvSplit+'NA'+'\n') -def transformGeneFile(purple_gene, sampleId, panelGenes): +def transformGeneFile(purple_gene, sampleId, panelGenes, output_dir='transformed_data'): first=True geneLocation = [] - ft=open('transformed_data/'+sampleId+'.transformed.genemetrics.cns','w') + ft=open(output_dir + '/' + sampleId + '.transformed.genemetrics.cns', 'w') with open(purple_gene,'rt') as fh: for lines in fh: lines = lines.rstrip('\n') @@ -151,9 +152,10 @@ def transformGeneFile(purple_gene, sampleId, panelGenes): ft.write(lines+tsvSplit+'1\n') return geneLocation -def genVCFfile(amber, sampleId): + +def genVCFfile(amber, sampleId, output_dir='transformed_data'): #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName - ft = open('transformed_data/'+sampleId+'.transformed.vcf','w') + ft = open(output_dir + '/' + sampleId + '.transformed.vcf', 'w') ft.write(vcf_header+"\n") first = True header = [] @@ -205,7 +207,7 @@ def getNormCorrection(cobalt_segmented): return weightedAverage(sValues,nProbes) -def getNormPurple(purple_segmented, purple_purity, sampleId): +def getNormPurple(purple_segmented, purple_purity): first = True header=[] @@ -229,11 +231,11 @@ def getNormPurple(purple_segmented, purple_purity, sampleId): Weights.append(int(arr[header.index('depthWindowCount')])) return weightedAverage(FClogs,Weights) -def transformSegmentedFile(purple_segmented,purple_purity, sampleId, normCorrection): +def transformSegmentedFile(purple_segmented,purple_purity, sampleId, normCorrection, output_dir='transformed_data'): # xx #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName - ft=open('transformed_data/' + sampleId + '.transformed.cns','w') + ft=open(output_dir + '/' + sampleId + '.transformed.cns', 'w') first = True header=[] @@ -284,6 +286,7 @@ class Config: purplePurity: str sampleId: str panelGenes: str + outputDir: str = "transformed_data" def parse_args(sys_args): parser = argparse.ArgumentParser(prog="TransformData", description="Transform hmftools output to input for reconCNV") @@ -297,8 +300,9 @@ def parse_args(sys_args): parser.add_argument("--purplePurity", "-p", type=str, required=True, help="purity file - ends with purple.purity.tsv") parser.add_argument("--sampleId","-i", type = str, required =True, help="sample Id - used for output file names") parser.add_argument("--panelGenes","-t", type = str, required = True, help="panel genes - used to annotate genes") + parser.add_argument("--outputDir","-o", type = str, required = False, help="output directory") args = parser.parse_args(sys_args) - return Config(args.cobaltRatio, args.cobaltSegmented, args.amber, args.purpledriverCatalog, args.purpleGene, args.purpleSomatic, args.purplePurity, args.sampleId, args.panelGenes) + return Config(args.cobaltRatio, args.cobaltSegmented, args.amber, args.purpledriverCatalog, args.purpleGene, args.purpleSomatic, args.purplePurity, args.sampleId, args.panelGenes, args.outputDir) def main(args): @@ -308,11 +312,13 @@ def main(args): panelGenes = getPanelGenes(args.panelGenes) normCorrection=getNormCorrection(args.cobaltSegmented) - geneLocation=transformGeneFile(args.purpleGene,sampleId, panelGenes) - transformRatioFile(args.cobaltRatio, sampleId, geneLocation) - genVCFfile(args.amber, sampleId) - normPurple=getNormPurple(args.purpleSomatic,args.purplePurity,sampleId) - transformSegmentedFile(args.purpleSomatic,args.purplePurity,sampleId,-normPurple+normCorrection) + geneLocation=transformGeneFile(args.purpleGene,sampleId, panelGenes, args.outputDir) + transformRatioFile(args.cobaltRatio, sampleId, geneLocation, args.outputDir) + genVCFfile(args.amber, sampleId, args.outputDir) + normPurple=getNormPurple(args.purpleSomatic, args.purplePurity) + transformSegmentedFile(args.purpleSomatic,args.purplePurity,sampleId,-normPurple+normCorrection, args.outputDir) + + if __name__ == "__main__": logging.basicConfig( format="%(asctime)s - [%(levelname)-8s] - %(message)s", level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S" From 3dfd8455b59dc9b6f18ef5e56f576bde1e14dfeb Mon Sep 17 00:00:00 2001 From: Erik van Dijk Date: Thu, 25 Apr 2024 16:06:07 +0200 Subject: [PATCH 2/5] Add files via upload --- panel/TransformData.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/panel/TransformData.py b/panel/TransformData.py index ff012e782..771384120 100644 --- a/panel/TransformData.py +++ b/panel/TransformData.py @@ -66,8 +66,19 @@ def getPurityPloidy(purple_purity): arr = lines.split(tsvSplit) res['purity'] = float(arr[header.index('purity')]) res['ploidy'] = float(arr[header.index('ploidy')]) + res['gender'] =str(arr[header.index('gender')]) return res +def writePurplePurity(purple_purity, sampleId, output_dir='transformed_data'): + res = getPurityPloidy(purple_purity) + + with open(output_dir + '/' + sampleId + '.purity', 'w') as ft: + ft.write(str(res['purity'])) + with open(output_dir + '/' + sampleId + '.gender', 'w') as ft: + ft.write(res['gender']) + with open(output_dir + '/' + sampleId + '.ploidy', 'w') as ft: + ft.write(str(res['ploidy'])) + def transformRatioFile(cobalt, sampleId,geneLocation, output_dir='transformed_data'): ## xx @@ -170,9 +181,9 @@ def genVCFfile(amber, sampleId, output_dir='transformed_data'): Depth = arr[header.index("tumorDepth")] BAF = arr[header.index("tumorBAF")] - SAF = str(int(round(int(Depth)*float(BAF)*0.5))) - SAR = str(int(float(Depth)*float(BAF)) - int(SAF)) + SAF = str(int(round(int(Depth)*float(BAF)*0.5))) + SAR = str(round(float(Depth)*float(BAF)) - int(SAF)) CHROM = arr[header.index('chromosome')] @@ -300,7 +311,7 @@ def parse_args(sys_args): parser.add_argument("--purplePurity", "-p", type=str, required=True, help="purity file - ends with purple.purity.tsv") parser.add_argument("--sampleId","-i", type = str, required =True, help="sample Id - used for output file names") parser.add_argument("--panelGenes","-t", type = str, required = True, help="panel genes - used to annotate genes") - parser.add_argument("--outputDir","-o", type = str, required = False, help="output directory") + parser.add_argument("--outputDir","-o", type = str, required = False, help="output directory",default = 'transformed_data') args = parser.parse_args(sys_args) return Config(args.cobaltRatio, args.cobaltSegmented, args.amber, args.purpledriverCatalog, args.purpleGene, args.purpleSomatic, args.purplePurity, args.sampleId, args.panelGenes, args.outputDir) @@ -317,7 +328,7 @@ def main(args): genVCFfile(args.amber, sampleId, args.outputDir) normPurple=getNormPurple(args.purpleSomatic, args.purplePurity) transformSegmentedFile(args.purpleSomatic,args.purplePurity,sampleId,-normPurple+normCorrection, args.outputDir) - + writePurplePurity(args.purplePurity,sampleId,args.outputDir) if __name__ == "__main__": logging.basicConfig( From 2e2a0d2ff2f1eba9ba83bcef92f002324a1fd60c Mon Sep 17 00:00:00 2001 From: Erik van Dijk Date: Wed, 22 May 2024 13:55:44 +0200 Subject: [PATCH 3/5] Changed config to use larger points and fix Also fixed a rounding error that led to slightly inaccurate BAF points in some cases. --- panel/TransformData.py | 2 +- panel/config.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/panel/TransformData.py b/panel/TransformData.py index 771384120..c7a1a7af5 100644 --- a/panel/TransformData.py +++ b/panel/TransformData.py @@ -134,7 +134,7 @@ def transformRatioFile(cobalt, sampleId,geneLocation, output_dir='transformed_da chr = data[0] gene = [geneLoc.gene for geneLoc in geneLocation if geneLoc.chr==data[0] and overlaps([geneLoc.start,geneLoc.end],[int(start),int(end)])] if gene: - ft.write(lines+tsvSplit+start+tsvSplit+end+tsvSplit+str(logR)+tsvSplit+';'.join(gene)+'\n') + ft.write(lines+tsvSplit+start+tsvSplit+end+tsvSplit+str(logR)+tsvSplit+';'.join(gene)+ '\n') else: ft.write(lines+tsvSplit+start+tsvSplit+end+tsvSplit+str(logR)+tsvSplit+'NA'+'\n') diff --git a/panel/config.json b/panel/config.json index 9e804e6b2..fc6ace64b 100644 --- a/panel/config.json +++ b/panel/config.json @@ -13,7 +13,7 @@ "field_separator": "\t", "off_target_label": "Antitarget", "off_target_low_conf_log2": -10, - "weight_scaling_factor": 10 + "weight_scaling_factor": 50 }, "genome_file":{ "column_names":{ @@ -81,6 +81,7 @@ "logFC_genome_plot":{ "width": 1200, "height": 250, + "point_size" : 4, "output_backend": "webgl", "active_scroll": "xwheel_zoom", "title": "Genome Level CNV", From c26d34ac8e6d861f25613769fa187f1cd5408a13 Mon Sep 17 00:00:00 2001 From: Mathijs den Burger Date: Wed, 22 May 2024 15:20:57 +0200 Subject: [PATCH 4/5] Document how to publish a preprocess-recon-cnv Docker image --- panel/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 panel/README.md diff --git a/panel/README.md b/panel/README.md new file mode 100644 index 000000000..883a4579b --- /dev/null +++ b/panel/README.md @@ -0,0 +1,22 @@ +# Publish new preprocess-recon-cnv release + +First make sure you can publish to the Hartwig Docker repository: + + gcloud auth configure-docker europe-west4-docker.pkg.dev + +Build the local Docker image: + + docker build -t preprocess-recon-cnv + +Test the local Docker image: + + docker run preprocess-recon-cnv + +If the local Docker image works as expected, tag it with the public name +and a [semantic version](https://semver.org) (we use `0.0.1` as the example version): + + docker tag preprocess-recon-cnv europe-west4-docker.pkg.dev/hmf-build/hmf-docker/preprocess-recon-cnv:0.0.1 + +Finally, push the Docker image to the Hartwig Docker repository: + + docker push europe-west4-docker.pkg.dev/hmf-build/hmf-docker/preprocess-recon-cnv:0.0.1 From 9b7e5af8a8d10600b024e3269b6d89c1ef5ab6c3 Mon Sep 17 00:00:00 2001 From: Mathijs den Burger Date: Fri, 24 May 2024 11:10:22 +0200 Subject: [PATCH 5/5] Enforce correct machine architecture in preprocess-recon-cnv --- panel/Dockerfile | 2 +- panel/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/panel/Dockerfile b/panel/Dockerfile index 643f522ab..17adccf3a 100644 --- a/panel/Dockerfile +++ b/panel/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM --platform=linux/amd64 python:3.12-slim WORKDIR /opt/app COPY . . ENTRYPOINT ["python", "TransformData.py"] \ No newline at end of file diff --git a/panel/README.md b/panel/README.md index 883a4579b..acf370ebd 100644 --- a/panel/README.md +++ b/panel/README.md @@ -6,7 +6,7 @@ First make sure you can publish to the Hartwig Docker repository: Build the local Docker image: - docker build -t preprocess-recon-cnv + docker build -t preprocess-recon-cnv . Test the local Docker image: