Skip to content

Commit

Permalink
Merge pull request #2416 from opencb/TASK-5858
Browse files Browse the repository at this point in the history 
TASK-5858 - Fix alignment and coverage index analyses in blob storage
  • Loading branch information
@jtarraga
jtarraga authored Apr 19, 2024
2 parents ea1648b + 313ae7f commit de9378c
Show file tree
Hide file tree
Showing 86 changed files with 491 additions and 189 deletions.
20 changes: 13 additions & 7 deletions ...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ protected void check() throws Exception {
super.check();

// Sanity check
if (StringUtils.isEmpty(getJobId())) {
throw new ToolException("Missing job ID");
}

if (StringUtils.isEmpty(getStudy())) {
throw new ToolException("Missing study when computing alignment coverage");
}
Expand Down Expand Up @@ -168,20 +172,22 @@ protected void run() throws Exception {
+ ") was not create, please, check log files.");
}

// Try to copy the BW file into the BAM file directory
// Try to move the BW file into the BAM file directory
boolean moveSuccessful = false;
Path targetPath = Paths.get(bamCatalogFile.getUri()).getParent().resolve(bwPath.getFileName());
try {
Files.move(bwPath, targetPath);
Path movedPath = Files.move(bwPath, targetPath);
moveSuccessful = targetPath.equals(movedPath);
} catch (Exception e) {
// Do nothing
logger.info("Moving from {} to {}: {}", bwPath, targetPath, e.getMessage());
// Log message
logger.info("Error moving the coverage file into the BAM folder {} to {}", bwPath, targetPath, e);
}

if (targetPath.toFile().exists()) {
if (moveSuccessful) {
bwPath = targetPath;
logger.info("Coverage file was copied into the BAM folder: {}", bwPath);
logger.info("Coverage file was moved into the BAM folder: {}", bwPath);
} else {
logger.info("Couldn't copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
logger.info("Couldn't move the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
bwPath);
}

Expand Down
70 changes: 36 additions & 34 deletions ...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,34 @@

package org.opencb.opencga.analysis.alignment;

import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.tools.alignment.BamManager;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.tools.OpenCgaTool;
import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.common.InternalStatus;
import org.opencb.opencga.core.models.file.*;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.tools.annotations.Tool;
import org.opencb.opencga.core.tools.annotations.ToolParams;

import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

@Tool(id = AlignmentIndexOperation.ID, resource = Enums.Resource.ALIGNMENT, description = "Index alignment.")
public class AlignmentIndexOperation extends OpenCgaTool {
public class AlignmentIndexOperation extends OpenCgaToolScopeStudy {

public static final String ID = "alignment-index-run";
public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file";

private String study;
private String inputFile;
@ToolParams
protected final AlignmentIndexParams indexParams = new AlignmentIndexParams();

private File inputCatalogFile;
private Path inputPath;
Expand All @@ -48,14 +53,23 @@ public class AlignmentIndexOperation extends OpenCgaTool {
protected void check() throws Exception {
super.check();

// Sanity check
if (StringUtils.isEmpty(getJobId())) {
throw new ToolException("Missing job ID");
}

if (StringUtils.isEmpty(getStudy())) {
throw new ToolException("Missing study when computing alignment index");
}

OpenCGAResult<File> fileResult;
try {
fileResult = catalogManager.getFileManager().get(getStudy(), inputFile, QueryOptions.empty(), token);
fileResult = catalogManager.getFileManager().get(getStudy(), indexParams.getFileId(), QueryOptions.empty(), token);
} catch (CatalogException e) {
throw new ToolException("Error accessing file '" + inputFile + "' of the study " + study + "'", e);
throw new ToolException("Error accessing file '" + indexParams.getFileId() + "' of the study " + study + "'", e);
}
if (fileResult.getNumResults() <= 0) {
throw new ToolException("File '" + inputFile + "' not found in study '" + study + "'");
throw new ToolException("File '" + indexParams.getFileId() + "' not found in study '" + study + "'");
}

inputCatalogFile = fileResult.getResults().get(0);
Expand All @@ -64,7 +78,7 @@ protected void check() throws Exception {

// Check if the input file is .bam or .cram
if (!filename.endsWith(AlignmentConstants.BAM_EXTENSION) && !filename.endsWith(AlignmentConstants.CRAM_EXTENSION)) {
throw new ToolException("Invalid input alignment file '" + inputFile + "': it must be in BAM or CRAM format");
throw new ToolException("Invalid input alignment file '" + indexParams.getFileId() + "': it must be in BAM or CRAM format");
}

outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION)
Expand All @@ -73,6 +87,9 @@ protected void check() throws Exception {

@Override
protected void run() throws Exception {
setUpStorageEngineExecutor(study);

logger.info("Running with parameters {}", indexParams);

step(ID, () -> {
// Compute index if necessary
Expand All @@ -82,51 +99,36 @@ protected void run() throws Exception {
bamManager.close();

if (!outputPath.toFile().exists()) {
throw new ToolException("Something wrong happened when computing index file for '" + inputFile + "'");
throw new ToolException("Something wrong happened when computing index file for '" + indexParams.getFileId() + "'");
}

// Try to copy the BAI file into the BAM file directory
// Try to move the BAI file into the BAM file directory
boolean moveSuccessful = false;
Path targetPath = inputPath.getParent().resolve(outputPath.getFileName());
try {
Files.move(outputPath, targetPath);
Path movedPath = Files.move(outputPath, targetPath);
moveSuccessful = targetPath.equals(movedPath);
} catch (Exception e) {
// Do nothing
logger.info("Moving from {} to {}: {}", outputPath, targetPath, e.getMessage());
// Log message
logger.info("Error moving from {} to {}", outputPath, targetPath, e);
}

if (targetPath.toFile().exists()) {
if (moveSuccessful) {
outputPath = targetPath;
logger.info("Alignment index file was copied into the BAM folder: {}", outputPath);
logger.info("Alignment index file was moved into the BAM folder: {}", outputPath);
} else {
logger.info("Couldn't copy the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
logger.info("Couldn't move the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
outputPath);
}

// Link generated BAI file and update samples info, related file
File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, token);
File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager,
token);

// Update BAM file internal in order to set the alignment index (BAI)
FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY),
baiCatalogFile.getId(), "HTSJDK library");
catalogManager.getFileManager().updateFileInternalAlignmentIndex(inputCatalogFile, fileAlignmentIndex, token);
});
}

public String getStudy() {
return study;
}

public AlignmentIndexOperation setStudy(String study) {
this.study = study;
return this;
}

public String getInputFile() {
return inputFile;
}

public AlignmentIndexOperation setInputFile(String inputFile) {
this.inputFile = inputFile;
return this;
}
}
31 changes: 16 additions & 15 deletions ...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,16 @@
import org.opencb.opencga.analysis.StorageManager;
import org.opencb.opencga.analysis.models.FileInfo;
import org.opencb.opencga.analysis.models.StudyInfo;
import org.opencb.opencga.analysis.tools.ToolRunner;
import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.catalog.utils.ParamUtils;
import org.opencb.opencga.core.api.ParamConstants;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.models.project.Project;
import org.opencb.opencga.core.models.study.Study;
Expand Down Expand Up @@ -87,21 +91,18 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto
initStatsMap();
}

//-------------------------------------------------------------------------
// INDEX
//-------------------------------------------------------------------------

public void index(String study, String inputFile, String outdir, String token) throws ToolException {
ObjectMap params = new ObjectMap();

AlignmentIndexOperation indexOperation = new AlignmentIndexOperation();
indexOperation.setUp(null, catalogManager, storageEngineFactory, params, Paths.get(outdir), jobId, token);

indexOperation.setStudy(study);
indexOperation.setInputFile(inputFile);

indexOperation.start();
}
// //-------------------------------------------------------------------------
// // INDEX
// //-------------------------------------------------------------------------
//
// public void index(String study, String inputFile, String outdir, String token) throws ToolException {
// ToolRunner toolRunner = new ToolRunner("", catalogManager, storageEngineFactory);
//
// AlignmentIndexParams params = new AlignmentIndexParams();
// params.setFileId(inputFile);
// toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, study), Paths.get(outdir),
// jobId, token);
// }

//-------------------------------------------------------------------------
// QUERY
Expand Down
Loading

0 comments on commit de9378c

Please sign in to comment.