pax_global_header00006660000000000000000000000064130132703340014506gustar00rootroot0000000000000052 comment=ce80f3a2af1d5d0d327c5b40562bd077477f80b0 HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/000077500000000000000000000000001301327033400203355ustar00rootroot00000000000000HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/CMakeLists.txt000066400000000000000000000070041301327033400230760ustar00rootroot00000000000000############## ### Header ### cmake_minimum_required (VERSION 2.8) project (HiLive) # Set the version number set (HiLive_VERSION_MAJOR 0) set (HiLive_VERSION_MINOR 3) # Set the k-mer length set (HiLive_K 15 CACHE INT "Set the k-mer length for index and mapper") # Set flags for compilation set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -g -pthread -W -Wall -std=gnu++11 -O0") # configure a header file to pass some of the CMake settings to the source code configure_file ( "${PROJECT_SOURCE_DIR}/lib/config.h.in" "${PROJECT_BINARY_DIR}/config.h" ) # add the binary tree to the search path for include files include_directories("${PROJECT_BINARY_DIR}") ############################# ### setup seqan libraries ### set (CMAKE_MODULE_PATH "/home/schulzeja/masterthesis/tools/seqan-src/util/cmake") # adjust this to [pathToSeqanCode]/util/cmake set (SEQAN_INCLUDE_PATH "/home/schulzeja/masterthesis/tools/seqan-src/include/") # adjust this to [pathToSeqanCode]/include # Configure SeqAn, enabling features for libbz2 and zlib. #set (SEQAN_FIND_DEPENDENCIES ZLIB BZip2) # original version from seqan tutorial set (SEQAN_FIND_DEPENDENCIES BZip2) find_package (SeqAn REQUIRED) # Add include directories, defines, and flags for SeqAn (and its dependencies). include_directories (${SEQAN_INCLUDE_DIRS}) add_definitions (${SEQAN_DEFINITIONS}) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SEQAN_CXX_FLAGS}") ############################# ### setup Boost libraries ### set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME ON) find_package( Boost COMPONENTS system filesystem program_options REQUIRED ) include_directories( ${Boost_INCLUDE_DIR} ) ############################ ### setup Zlib and lz4 libraries ### find_package( ZLIB REQUIRED ) set (LZ4_PATH /usr/local/lib) # possibly adjust this to [pathToLz4]/lib if using downloaded lz4 source code include_directories(${LZ4_PATH}) link_directories(${LZ4_PATH}) set(CompressionLibs "${ZLIB_LIBRARIES};lz4") ############################################# ### download & build the Samtools library ### file(DOWNLOAD "https://sourceforge.net/projects/samtools/files/samtools/1.2/samtools-1.2.tar.bz2/download" ${CMAKE_CURRENT_BINARY_DIR}/samtools-1.2.tar.bz2) execute_process(COMMAND tar xjf ${CMAKE_CURRENT_BINARY_DIR}/samtools-1.2.tar.bz2 ) execute_process(COMMAND make -s -C samtools-1.2 ) include_directories( ${CMAKE_CURRENT_BINARY_DIR}/samtools-1.2 ) include_directories( ${CMAKE_CURRENT_BINARY_DIR}/samtools-1.2/htslib-1.2.1 ) link_directories( ${CMAKE_CURRENT_BINARY_DIR}/samtools-1.2 ${CMAKE_CURRENT_BINARY_DIR}/samtools-1.2/htslib-1.2.1 ) ############################## ### setup HiLive libraries ### include_directories("${PROJECT_SOURCE_DIR}/lib") # make a list of HiLive libraries set (LIB_NAMES tools alnread alnstream illumina_parsers kindex parallel argument_parser) set(LIB_LIST "") foreach (x ${LIB_NAMES}) list(APPEND LIB_LIST "lib/${x}.cpp") endforeach() add_library(HiLiveLibs ${LIB_LIST}) ############################# ### Build the executables ### add_executable (hilive tools/hilive.cpp) target_link_libraries (hilive HiLiveLibs ${CompressionLibs} ${Boost_LIBRARIES} ${SEQAN_LIBRARIES} hts) add_executable(hilive-build tools/build_index.cpp ) target_link_libraries(hilive-build HiLiveLibs ${CompressionLibs} ${Boost_LIBRARIES} ${SEQAN_LIBRARIES}) ##################### ### for debugging ### #get_cmake_property(_variableNames VARIABLES) #foreach (_variableName ${_variableNames}) #message(STATUS "${_variableName}=${${_variableName}}") #endforeach() HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/CONTRIBUTORS000066400000000000000000000003551301327033400222200ustar00rootroot00000000000000Bernhard Y. Renard * Project head Martin S. Lindner * Project founder, implemented versions 0.1 and 0.2 Jakob M. Schulze * developed version 0.2 to 0.3 HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/LICENSE000066400000000000000000000030221301327033400213370ustar00rootroot00000000000000Copyright (c) 2015-2016, Martin S. Lindner and the HiLive contributors. See CONTRIBUTORS for more info. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/README.md000066400000000000000000000113111301327033400216110ustar00rootroot00000000000000HiLive - Live Mapping of Illumina reads ======================================= Description ----------- HiLive is a read mapping tool that maps Illumina HiSeq (or comparable) reads right in the moment when they are produced. This means, read mapping is finished as soon as the sequencer is finished. Website ------- The HiLive project website is https://gitlab.com/SimonHTausch/HiLive There you can find the latest version of HiLive, source code, documentation, and examples. Installation ------------ If you are using a Debian based system you can directly install the Debian package from the official [Debian repository](https://packages.debian.org/sid/hilive "HiLive Debian package") If this does not work for you, you can still compile HiLive from source. Make sure that the following dependencies are installed: * cmake (>= 2.8) * boost (system, filesystem, program\_options) * zlib * lz4 If using a local version of lz4 then adjust path in CMakeLists.txt line 61. --- You also need to download a specific seqan build. Navigate to a folder where you want to store the seqan library and run: git clone https://github.com/seqan/seqan.git cd seqan && git checkout 9119aa6 --- Check out the HiLive source code from the project website and adjust the paths of the seqan module in the file CMakeLists.txt in the hilive folder (line 31 and 32). Then, compile HiLive with: cd [hilive-code] mkdir build && cd build cmake .. make --- To compile HiLive with a different k-mer size than k=15 make the following adjustment (here: k=10): cmake -DHiLive\_K 10 .. make Usage ----- HiLive has two components: * ``hilive-build`` builds the k-mer index of the reference genome * ``hilive`` the read mapper itself --- #### Using hilive-build: Building a k-mer index from FASTA file input.fa to output file input.fa.kix: hilive-build input.fa Building an index from a large reference genome. Here is makes sense to use trimming, i.e. removing k-mers from the index that occurr more than 1000 times (for example) in the index. The index is written into the file trimmed.kix hilive-build -t 1000 -o trimmed.kix input.fa --- #### Using hilive: To map reads in a 100bp run using default settings: hilive /path/to/BaseCalls /path/to/index.kix 100 /path/to/outputFolder Hint: to concatenate the resulting sam files of a lane to a single file using samtools you could use the following commands: for file in /path/to/outputFolder/L001/s_1_*.sam; do samtools view -bS $file > ${file%.sam}.bam; done samtools cat /path/to/outputFolder/L001/s_1_*.bam > /path/to/outputFolder/L001/allAlignments.bam samtools view -SH /path/to/outputFolder/L001/s_1_1101.sam > /path/to/outputFolder/L001/allAlignments.sam samtools view /path/to/outputFolder/L001/allAlignments.bam >> /path/to/outputFolder/L001/allAlignments.sam --- #### Demultiplexing: To map reads from multiplexed sequencing runs, you can provide HiLive with the barcode sequences from your Sample Sheet. In default cases, barcode sequences are read after the (first) read, such that demultiplexing is carried out after the mapping is completed. If you use double indexing, please concatenate both indices in the correct order and provide them as one sequence. Please take care that the number of cycles is exactly the read length from your Sample Sheet plus that of your complete barcode sequence. All entered indices must be of the same length. To provide multiple indices, enter the -XXX argument for every barcode or barcode combination, e.g.: hilive /path/to/BaseCalls /path/to/index.kix 107 /path/to/outputFolder -b barcode1 -b barcode2 ... Reads containing one of the given barcodes will be written to the resulting samfiles. The corresponding barcode sequence is stored in the BC-field of the samfile. Hint: You can write the results of each sample into a separate files using the following command: for barcode in barcode1 barcode2 ... ; do grep "^@" allAlignments.sam > allAlignments_$barcode.sam && grep "BC:Z:$barcode" allAlignments.sam >> allAlignments_$barcode.sam; done --- We recommend to adjust the numbers of threads used by HiLive with -n. If possible, make use of all threads on the machine! Please consult the project website for more details on the parameters! License ------- See the file LICENSE for licensing information. Contact ------- Please consult the HiLive project website for questions! If this does not help, please feel free to consult: * [Martin S. Lindner](mailto:martin@mail-lindner.de "Mail Martin S. Lindner") (technical contact) * [Bernhard Y. Renard](mailto:renardb@rki.de "Mail Bernhard Y. Renard") (project head) also see CONTRIBUTORS for a complete list of contributors and their contact information HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/000077500000000000000000000000001301327033400211035ustar00rootroot00000000000000HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/alnread.cpp000066400000000000000000000710331301327033400232210ustar00rootroot00000000000000#include "alnread.h" seqan::String > Seed::returnSeqanCigarString() { typedef seqan::String > TSeqanCigarString; TSeqanCigarString seqanCigarString; seqan::CigarElement<> cigarElem; int last_offset = 0; for (CigarVector::const_iterator it = cigar_data.begin(); it != cigar_data.end(); ++it) { // for every element in cigar_data if (it == cigar_data.begin() && (*it).offset==NO_MATCH) { // Alignment begins with NO_MATCH region => Softclipped start cigarElem.operation='S'; cigarElem.count=(*it).length; seqan::appendValue(seqanCigarString, cigarElem); continue; } if (it == --cigar_data.end() && (*it).offset==NO_MATCH) { // Alignment ends with NO_MATCH region => Softclipped end cigarElem.operation='S'; cigarElem.count=(*it).length; seqan::appendValue(seqanCigarString, cigarElem); continue; } if ((*it).offset==NO_MATCH) { // ??? -> Mismatch cigarElem.operation='M'; cigarElem.count=(*it).length; seqan::appendValue(seqanCigarString, cigarElem); continue; } if ((*it).offset!=NO_MATCH) { // ??? -> Match if (last_offset == (*it).offset) { // no offset change cigarElem.operation='M'; cigarElem.count=(*it).length; seqan::appendValue(seqanCigarString, cigarElem); last_offset = (*it).offset; continue; } if (last_offset < (*it).offset) { // offset gets bigger => reference in alignment is longer than read => deletion in read (and thereby cigar string) cigarElem.operation='D'; cigarElem.count=(*it).offset - last_offset; seqan::appendValue(seqanCigarString, cigarElem); cigarElem.operation='M'; cigarElem.count=(*it).length; seqan::appendValue(seqanCigarString, cigarElem); last_offset = (*it).offset; continue; } if (last_offset > (*it).offset) { // offset gets smaller => reference in alignment is smaller than read => insertion in read (and thereby cigar string) cigarElem.operation='I'; cigarElem.count=last_offset - (*it).offset; seqan::appendValue(seqanCigarString, cigarElem); cigarElem.operation='M'; cigarElem.count=(*it).length; seqan::appendValue(seqanCigarString, cigarElem); last_offset = (*it).offset; continue; } } } // collapse Neighboring match regions for (unsigned k = 1; k= 256) throw std::overflow_error("CIGAR information contains more than 255 elements!"); uint8_t cigar_len = cigar_data.size(); total_size += sizeof(uint8_t); // the size of the cigar information total_size += cigar_len*(sizeof(CountType) + sizeof(DiffType)); // the cigar information itself return total_size; } std::vector Seed::serialize() { // get the total size of the serialization uint16_t total_size = serialize_size(); uint8_t cigar_len = (uint8_t) cigar_data.size(); // create the vector to store the data std::vector data (total_size); char* d = data.data(); // write the target Genome ID memcpy(d,&gid,sizeof(GenomeIdType)); d += sizeof(GenomeIdType); // write the start position memcpy(d,&start_pos,sizeof(PositionType)); d += sizeof(PositionType); // write the number of matches memcpy(d,&num_matches,sizeof(CountType)); d += sizeof(CountType); // write the number of cigar elements memcpy(d,&cigar_len,sizeof(uint8_t)); d += sizeof(uint8_t); // write the seeds for (auto it = cigar_data.begin(); it != cigar_data.end(); ++it) { memcpy(d,&(it->length),sizeof(CountType)); d += sizeof(CountType); memcpy(d,&(it->offset),sizeof(DiffType)); d += sizeof(DiffType); } return data; } uint16_t Seed::deserialize(char* d) { // the total number of bytes read uint16_t bytes = 0; // read the target Genome ID memcpy(&gid,d,sizeof(GenomeIdType)); bytes += sizeof(GenomeIdType); // read the start position memcpy(&start_pos,d+bytes,sizeof(PositionType)); bytes += sizeof(PositionType); // read the number of matches memcpy(&num_matches,d+bytes,sizeof(CountType)); bytes += sizeof(CountType); // read the number of cigar elements uint8_t cigar_len = 0; memcpy(&cigar_len,d+bytes,sizeof(uint8_t)); bytes += sizeof(uint8_t); // read the cigar elements cigar_data.clear(); for (uint8_t i = 0; i < cigar_len; ++i) { CigarElement cig; memcpy(&(cig.length),d+bytes,sizeof(CountType)); bytes += sizeof(CountType); memcpy(&(cig.offset),d+bytes,sizeof(DiffType)); bytes += sizeof(DiffType); cigar_data.emplace_back(cig); } return bytes; } void ReadAlignment::set_rlen(CountType r) { rlen = r; } uint64_t ReadAlignment::serialize_size() { // calculate total size first uint64_t total_size = 0; total_size += 1; // the flag total_size += sizeof(CountType); // the cycle number total_size += sizeof(CountType); // the last_invalid cycle total_size += sizeof(CountType); // the sequence length total_size += sequenceStoreVector.size()*(sizeof(uint8_t)); // the sequence information itself // total number of seeds total_size += sizeof(uint32_t); // size of the single seeds for (auto & s : seeds) { total_size += sizeof(uint16_t) + s->serialize_size(); } return total_size; } std::vector ReadAlignment::serialize() { // get the total size of the serialization uint64_t total_size = serialize_size(); uint32_t num_seeds = (uint32_t) seeds.size(); // create the vector to store the data std::vector data (total_size); char* d = data.data(); // write the flag memcpy(d,&flags,1); d++; // write the cycle memcpy(d,&cycle,sizeof(CountType)); d += sizeof(CountType); // write the last invalid cycle memcpy(d,&last_invalid,sizeof(CountType)); d += sizeof(CountType); // write the sequence length memcpy(d,&sequenceLen,sizeof(CountType)); d += sizeof(CountType); // write the sequenceStoreVector for (auto it = sequenceStoreVector.begin(); it != sequenceStoreVector.end(); ++it) { memcpy(d,&(*it),sizeof(uint8_t)); d += sizeof(uint8_t); } // write the number of seeds memcpy(d,&num_seeds,sizeof(uint32_t)); d += sizeof(uint32_t); // write the seeds for (auto it = seeds.begin(); it != seeds.end(); ++it) { std::vector seed_data = (*it)->serialize(); uint16_t seed_size = seed_data.size(); memcpy(d,&seed_size,sizeof(uint16_t)); d += sizeof(uint16_t); memcpy(d,seed_data.data(),seed_size); d += seed_size; } return data; } uint64_t ReadAlignment::deserialize(char* d) { // the total number of bytes read uint64_t bytes = 0; // read the flag memcpy(&flags,d,1); bytes++; // read the cycle memcpy(&cycle,d+bytes,sizeof(CountType)); bytes += sizeof(CountType); // read the last invalid cycle memcpy(&last_invalid,d+bytes,sizeof(CountType)); bytes += sizeof(CountType); // read the sequence length sequenceLen = 0; memcpy(&sequenceLen,d+bytes,sizeof(CountType)); bytes += sizeof(CountType); // read the sequence unsigned seqVec_size = (unsigned) std::ceil((float) sequenceLen / 4.0); sequenceStoreVector.clear(); sequenceStoreVector.reserve(seqVec_size); for (unsigned i = 0; i seed_data (seed_size,0); memcpy(seed_data.data(),d+bytes,seed_size); bytes += seed_size; USeed s (new Seed); s->deserialize(seed_data.data()); // insert into sorted list. The data read in is already sorted!!! // therefore I only push back seeds.push_back(std::move(s)); } return bytes; } // convert and return sequence of the seed as string std::string ReadAlignment::getSequenceString(AlignmentSettings & settings) { std::string seq = ""; // append one 4 base block at a time for (unsigned i = 0; i new 8 Bit block needs to be created uint8_t newBlock = twobit_repr(nuc); newBlock = newBlock << 6; // 'empty' bits are on the right side sequenceStoreVector.push_back(newBlock); ++sequenceLen; } else { // not all bits are used. At least the last two are unoccupied // append new 2 bits to the right of the old bits if (sequenceLen % 4 == 1) // 6 bits empty sequenceStoreVector.back() = sequenceStoreVector.back() >> 4; if (sequenceLen % 4 == 2) // 4 bits empty sequenceStoreVector.back() = sequenceStoreVector.back() >> 2; sequenceStoreVector.back() = sequenceStoreVector.back() | twobit_repr(nuc); ++sequenceLen; // shift used bits until they are left aligned if (sequenceLen % 4 == 2) // 4 bits empty sequenceStoreVector.back() = sequenceStoreVector.back() << 4; if (sequenceLen % 4 == 3) // 2 bits empty sequenceStoreVector.back() = sequenceStoreVector.back() << 2; } } // helper function for add_new_seeds bool seed_compare_pos (const USeed & i, const USeed & j) { return (i->start_pos < j->start_pos); } // Create new seeds from a list of kmer positions and add to current seeds void ReadAlignment::add_new_seeds(GenomePosListType& pos, std::vector & posWasUsedForExtension, AlignmentSettings & settings) { SeedVecIt sit = seeds.begin(); for(GenomePosListIt it = pos.begin(); it != pos.end(); ++it) { if (posWasUsedForExtension[it - pos.begin()]) // if current reference hit was used at least once for seed extension continue; USeed s (new Seed); s->gid = it->gid; s->start_pos = it->pos - (cycle-settings.kmer_span); s->num_matches = K_HiLive; s->cigar_data.clear(); if (cycle-settings.kmer_span > 0) s->cigar_data.emplace_back(cycle-settings.kmer_span,NO_MATCH); // set correct matches and mismatches depending on kmer mask std::vector gapVec = settings.kmer_gaps; gapVec.push_back(settings.kmer_span+1); unsigned lastProcessedGapPosition = 0; for (unsigned gapIndex = 0, nextGap; gapIndex < gapVec.size(); ++gapIndex) { nextGap = gapVec[gapIndex]; if (nextGap - lastProcessedGapPosition - 1 > 0) s->cigar_data.emplace_back(nextGap - lastProcessedGapPosition - 1,0); lastProcessedGapPosition = nextGap; s->cigar_data.emplace_back(1,NO_MATCH); } s->cigar_data.pop_back(); // remove tailing NO_MATCH from the for-loop // insert seed into sorted list of seeds while (sit != seeds.end() && seed_compare_pos(*sit, s)) // if seed exists and elem has larger starting position than (*sit) ++sit; sit = seeds.insert(sit, std::move(s)); } } // Extend or create a placeholder seed for read with only trimmed matches void ReadAlignment::create_placeholder_seed(AlignmentSettings & settings) { // if no seed exist, create one (only in the correct cycle and if parameter is true). if(seeds.size()==0){ USeed s (new Seed); s->gid = TRIMMED; s->num_matches = 1; s->cigar_data.clear(); s->cigar_data.emplace_back(settings.kmer_span,TRIMMED_MATCH); // this is always sorted, because there is only one seed now seeds.push_back(std::move(s)); } } // convert a placeholder seed to a set of normal seeds void ReadAlignment::convertPlaceholder(GenomePosListType& pos, AlignmentSettings & settings){ if(seeds.size()!=1) return; auto s = seeds.begin(); auto matches = (*s)->num_matches; if(cycle - settings.kmer_span > 0 && (*s)->gid==TRIMMED){ seeds.clear(); for(auto p = pos.begin(); p != pos.end(); ++p){ USeed newSeed (new Seed()); newSeed->gid = p->gid; newSeed->start_pos = p->pos - (cycle - settings.kmer_span); newSeed->num_matches = matches; newSeed->cigar_data.clear(); newSeed->cigar_data.emplace_back(cycle - settings.kmer_span,NO_MATCH); // set correct matches and mismatches depending on kmer mask std::vector gapVec = settings.kmer_gaps; gapVec.push_back(settings.kmer_span); // without +1 because the last match will be inserted in extend_alignment unsigned lastProcessedGapPosition = 0; for (unsigned gapIndex = 0, nextGap; gapIndex < gapVec.size(); ++gapIndex) { nextGap = gapVec[gapIndex]; if (nextGap - lastProcessedGapPosition - 1 > 0) newSeed->cigar_data.emplace_back(nextGap - lastProcessedGapPosition - 1,0); lastProcessedGapPosition = nextGap; newSeed->cigar_data.emplace_back(1,NO_MATCH); } newSeed->cigar_data.pop_back(); // remove tailing NO_MATCH from the for-loop // insert into sorted list. The list is empty and pos vector is already sorted. // therefore I only push back seeds.push_back(std::move(newSeed)); } } } // filter seeds based on filtering mode and q gram lemma. Also calls add_new_seeds. void ReadAlignment::filterAndCreateNewSeeds(AlignmentSettings & settings, GenomePosListType & pos, std::vector & posWasUsedForExtension) { // TODO this function misbehaves when using demultiplexing. Reads might be preferred, kicking others only to get discarded due to barcode in the end. // compute possible remaining matches int possibleRemainingMatches = settings.seqlen - cycle + settings.kmer_span - 1; if (cycle == settings.seqlen) possibleRemainingMatches = 0; // compute threshold for number of matching bases a seed must already have to have a chance of getting printed in the end // adjust threshold via adapted q-gram-lemma int num_matches_threshold = settings.seqlen - settings.min_errors*(settings.kmer_span) - possibleRemainingMatches; if ((settings.all_best_hit_mode) || (settings.any_best_hit_mode)) { int max_num_matches = 0; for(SeedVecIt sd = seeds.begin() ; sd !=seeds.end(); ++sd) max_num_matches = std::max(max_num_matches, (int) (*sd)->num_matches); num_matches_threshold = std::max(num_matches_threshold, (int) max_num_matches - possibleRemainingMatches); } if (settings.all_best_n_scores_mode && settings.best_n > 0) { std::vector num_matches_vector; for (SeedVecIt it=seeds.begin(); it!=seeds.end();++it) num_matches_vector.push_back((*it)->num_matches); std::sort(num_matches_vector.begin(), num_matches_vector.end(), std::greater()); // sort in decreasing order unsigned numberOfUniques = std::unique(num_matches_vector.begin(), num_matches_vector.end()) - num_matches_vector.begin(); // uniques are in front of vector now if (numberOfUniques > settings.best_n) { CountType nth_best_num_matches = num_matches_vector[settings.best_n - 1]; num_matches_threshold = std::max(num_matches_threshold, (int) nth_best_num_matches - possibleRemainingMatches); } } // else nothing has to be done for all-hit-mode // delete all seeds which do not reach threshold SeedVecIt it=seeds.begin(); bool foundHit = false; while ( it!=seeds.end()) { if ((*it)->num_matches < num_matches_threshold) it = seeds.erase(it); else if (settings.discard_ohw && (cycle>settings.start_ohw) && ((*it)->num_matches <= K_HiLive)) // remove one-hit-wonders it = seeds.erase(it); else if (cycle == settings.seqlen && settings.any_best_hit_mode && foundHit) it = seeds.erase(it); else ++it; foundHit = true; } // if a new seed would have a chance then create it if (num_matches_threshold <= 1) // new seed would have 1 more match than expected by possibleRemainingMatches add_new_seeds(pos, posWasUsedForExtension, settings); } // updates cigar_data accordingly to a new matching kmer void ReadAlignment::addMatchingKmer(USeed & s, DiffType offset, AlignmentSettings & settings) { s->cigar_data.emplace_back(1,offset); s->num_matches += 1; //////////////////////////////////////////////////////////// //// determine last occurred offset //////////////////////// int last_offset = 0; if ((*prev(prev(s->cigar_data.end()))).offset != NO_MATCH) // if last CigarElement is Match last_offset = (*prev(prev(s->cigar_data.end()))).offset; else // then the one before has to be a match last_offset = (*prev(prev(prev(s->cigar_data.end())))).offset; assert(last_offset != NO_MATCH); //////////////////////////////////////////////////////////// //// split last kmer-span bases in single CigarElements //// CigarVector::iterator it = --(s->cigar_data.end()); unsigned summedLength = 1; while (summedLength < settings.kmer_span) { ++summedLength; --it; if ((*it).length > 1) { CigarElement elem1(1, (*it).offset); CigarElement elem2((*it).length-1, (*it).offset); s->cigar_data.insert(it, elem2); s->cigar_data.insert(it, elem1); it = s->cigar_data.erase(it); // points now at element after former it --it; // points now at elem1 } } // it points now at kmer-spans-th last element of cigar_data list //////////////////////////////////////////////////////////// //// remove possibly inserted bases //////////////////////// if (offset < last_offset) { --it; for (int i=0; i 1) --((*it).length); else it = --(s->cigar_data.erase(it)); } ++it; // now again points at kmer-spans-th last element of cigar_data list } //////////////////////////////////////////////////////////// //// set matched bases as match //////////////////////////// CigarVector::iterator it_save = it; // for joining unsigned positionInKmer = 1; while (it != s->cigar_data.end()) { // if positionInKmer is not in kmer_gaps and cigar element was match if ((*it).offset == NO_MATCH && std::find(settings.kmer_gaps.begin(), settings.kmer_gaps.end(), positionInKmer) == settings.kmer_gaps.end()) { (*it).offset = offset; s->num_matches += 1; } ++it; ++positionInKmer; } //////////////////////////////////////////////////////////// //// join last kmer-span+1 CigarElements /////////////////// it = it_save; if (it == s->cigar_data.begin()) ++it; while (it != s->cigar_data.end()) { if ((*prev(it)).offset == (*it).offset) { (*prev(it)).length += 1; it = s->cigar_data.erase(it); } else ++it; } } // Extend an existing seed (to be precise, extend the CIGAR vector / data). bool ReadAlignment::extendSeed(USeed & s, DiffType offset, AlignmentSettings & settings){ // Extend CIGAR for TRIMMED k-mers if ((offset == TRIMMED_MATCH) || ((offset == NO_MATCH) && (s->cigar_data.back().offset == TRIMMED_MATCH))){ s->cigar_data.back().length += 1; if (s->cigar_data.back().offset != NO_MATCH){ s->num_matches += 1; return true; } return false; } // if last region was match region if(s->cigar_data.back().offset != NO_MATCH){ // MATCH -> NO_MATCH if(offset == NO_MATCH){ // new mismatch region s->cigar_data.emplace_back(1,NO_MATCH); return false; } // MATCH -> MATCH Extend match region with new match. else { int offset_change = offset - s->cigar_data.rbegin()->offset; // without any mismatch in between there cannot be a valid offset_change other than 0 if (offset_change!=0) { // new mismatch region s->cigar_data.emplace_back(1,NO_MATCH); return false; } // else: extend current match region addMatchingKmer(s, offset, settings); return true; } } // so last region was mismatch region else { // NO_MATCH -> NO_MATCH Extend mismatch region with new mismatch. if(offset == NO_MATCH){ s->cigar_data.back().length += 1; return false; } // NO_MATCH -> MATCH else { assert((++(s->cigar_data.rbegin()))->offset != TRIMMED_MATCH && (++(s->cigar_data.rbegin()))->offset != NO_MATCH); int offset_change = offset - (++(s->cigar_data.rbegin()))->offset; // If there is an offset change, I need to have seen the appropriate mismatches before. if ( offset_change == 0 || ((offset_change < 0) && (s->cigar_data.back().length >= -offset_change + settings.kmer_span - 1)) // Insertion in read || ((offset_change > 0) && (s->cigar_data.back().length >= settings.kmer_span - 1 )) ) { // Deletion in read addMatchingKmer(s, offset, settings); return true; } else { // criteria not fulfilled: extend existing mismatch area. s->cigar_data.back().length += 1; return false; } } } } void ReadAlignment::extend_alignment(char bc, KixRun* index, AlignmentSettings* settings) { // move to the next cycle cycle += 1; // update the last k-mer uint8_t qual = ((bc >> 2) & 63); // get bits 3-8 if ( (bc == 0) || (qual < settings->min_qual) ){ // no call if all 0 bits or quality below threshold last_invalid = cycle; // TODO append an N as basecall? Could be a bad idea } unsigned mask = 3; if (flags != 0) // if read is valid appendNucleotideToSequenceStoreVector(revtwobit_repr(bc & mask)); // get the nucleotide as an actual character, disregarding the quality // do not update the alignments when reading barcode if (cycle > settings->seqlen) { if (cycle == rlen) if (std::find(settings->barcodeVector.begin(), settings->barcodeVector.end(), getBarcodeString(*settings)) == settings->barcodeVector.end()) this->disable(*settings); return; } // do not update the alignments when reading the first kmer_span-1 cycles if (cycle < settings->kmer_span) return; // update the alignments GenomePosListType pos; std::vector posWasUsedForExtension; // if last kmer of read is not valid if (!( last_invalid+settings->kmer_span-1 < cycle )) { // write a NO_MATCH for (auto sit = seeds.begin(); sit != seeds.end(); ++sit) extendSeed(*sit, NO_MATCH, *settings); } else { // get all occurrences of last_kmer (fwd & rc) from index const std::string sequence = getSequenceString(*settings); std::string::const_iterator it_lastKmer = sequence.end() - settings->kmer_span; HashIntoType last_kmer = 0; hash_fw(it_lastKmer, sequence.end(), last_kmer, *settings); pos = index->retrieve_positions(sequence.substr(sequence.length()-settings->kmer_span), *settings); posWasUsedForExtension.resize(pos.size(), false); // check if the current k-mer was trimmed in the index if ( (pos.size() == 1) && ((*pos.begin()).gid == TRIMMED) ) { if(seeds.size()==0 && cycle==settings->kmer_span ){ // Create placeholder seed if first k-mer is trimmed create_placeholder_seed(*settings); } else // pretend that all existing seeds could be extended for(auto sd = seeds.begin() ; sd !=seeds.end(); ++sd) extendSeed(*sd, TRIMMED_MATCH, *settings); // clear the pos list so nothing bad happens in the next steps pos.clear(); } // not trimmed in the index --> try to extend existing seeds else { // Convert placeholder seeds (if any) to "real" seeds if(seeds.size()==1 && (*(seeds.begin()))->gid==TRIMMED){ convertPlaceholder(pos, *settings); } // find support for each candidate: iterate over seed candidates and positions simultaneously auto cPos1 = pos.begin(), cPos2 = pos.begin(); // sliding window [cPos1, cPos2) for (auto cSeed = seeds.begin(); cSeed!=seeds.end(); ++cSeed ) { PositionType seed_pos = (*cSeed)->start_pos + cycle -settings->kmer_span; // adjust the window in the position list while( (cPos1!=pos.end()) && (cPos1->pos < seed_pos - settings->window) ) ++cPos1; while( (cPos2!=pos.end()) && (cPos2->pos <= seed_pos + settings->window) ) ++cPos2; // search all positions in the window for the best matching extension of the seed DiffType best_offset = settings->window+1; // set larger than search window GenomePosListIt best_match = cPos2; // set behind the last element of the window for(GenomePosListIt kmerHitIt = cPos1; kmerHitIt!=cPos2; ++kmerHitIt) if (kmerHitIt->gid == (*cSeed)->gid){ // if offset gets bigger => Deletion in read int offset = kmerHitIt->pos - seed_pos; if ((best_match==cPos2)||(abs(offset) < abs(best_offset))) { best_match = kmerHitIt; best_offset = offset; } } // check if a best match was found for this seed if (best_match != cPos2) { if(extendSeed(*cSeed, best_offset, *settings)) // if pos was used as match, mark it so that later it does not get converted into a new seed posWasUsedForExtension[best_match-pos.begin()] = true; } else{ // no position found to extend the current seed extendSeed(*cSeed, NO_MATCH, *settings); } } // END: for(seeds...) } // END: not trimmed } // END: if last kmer is valid filterAndCreateNewSeeds(*settings, pos, posWasUsedForExtension); return; } // disable this alignment, i.e. delete all seeds and set the last_invalid indicator to the // end of the read. --> This read will not be aligned and consumes almost no space. void ReadAlignment::disable(AlignmentSettings & settings) { last_invalid = settings.seqlen; seeds.clear(); flags = 0; sequenceLen=0; sequenceStoreVector.clear(); } // obtain start position of a seed according to SAM (leftmost) PositionType ReadAlignment::get_SAM_start_pos(USeed & sd, AlignmentSettings & settings) { PositionType pos = sd->start_pos; if (pos < 0) { if (sd->cigar_data.back().offset == NO_MATCH) pos = -pos - settings.seqlen + settings.kmer_span - (++sd->cigar_data.rbegin())->offset; else pos = -pos - settings.seqlen + settings.kmer_span - (sd->cigar_data.rbegin())->offset; } return pos; } // Calculate the mapping quality for all alignments of the read based on the other alignments and the number of matching positions. int16_t MAPQ(const SeedVec &sv){ return sv.size(); } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/alnread.h000066400000000000000000000111001301327033400226530ustar00rootroot00000000000000#ifndef ALNREAD_H #define ALNREAD_H #include "headers.h" #include "definitions.h" #include "kindex.h" #include "tools.h" #include #include //-------------------------------------------------------------------// //------ The Seed data structure ----------------------------------// //-------------------------------------------------------------------// // a Seed stores the alignment of a read to a target genome struct Seed { // internal sequence ID of taget genome GenomeIdType gid; // (estimated) start position of the read on the target PositionType start_pos; // number of matching bases CountType num_matches; // Information about matches/mismatches (similar to CIGAR). The last element is the current one CigarVector cigar_data; // return Seqans String of CigarElement seqan::String > returnSeqanCigarString(); // get the size of the serialized object uint16_t serialize_size(); // serialize the object std::vector serialize(); // deserialize (read) data from a char vector uint16_t deserialize(char* d); }; typedef std::unique_ptr USeed; // compare function to sort Seed objects by position bool seed_compare_pos (const USeed & i, const USeed & j); // std::list of Seed pointers is much faster typedef std::list SeedVec; // a SeedVec Iterator typedef SeedVec::iterator SeedVecIt; //-------------------------------------------------------------------// //------ The Read-Alignment class ---------------------------------// //-------------------------------------------------------------------// class ReadAlignment { private: // read length CountType rlen; // sequence of the read so far, saved as vector so interpretation is not that trivial. Contains barcode CountType sequenceLen=0; std::vector sequenceStoreVector; // Extend or create a placeholder seed for read with only trimmed matches void create_placeholder_seed(AlignmentSettings & settings); // convert a placeholder seed to a set of normal seeds void convertPlaceholder(GenomePosListType& pos, AlignmentSettings & settings); // Create new seeds from a list of kmer positions and add to current seeds void add_new_seeds(GenomePosListType& pos, std::vector & posWasUsedForExtension, AlignmentSettings & settings); // filter seeds based on filtering mode and q gram lemma. Also calls add_new_seeds. void filterAndCreateNewSeeds(AlignmentSettings & settings, GenomePosListType & pos, std::vector & posWasUsedForExtension); // updates cigar_data accordingly to a new matching kmer void addMatchingKmer(USeed & s, DiffType offset, AlignmentSettings & settings); // Extend an existing CIGAR string for a seed based on a new basecall. return false if last CIGAR element after extension is mismatch area (NO_MATCH), true otherwise. bool extendSeed(USeed & s, DiffType offset, AlignmentSettings & settings); public: // have everything public until the apropriate access functions are available // Flags for this read; 1 = read is valid (illumina flag) unsigned char flags = 1; // the last invalid cycle CountType last_invalid; // the current cycle CountType cycle; // a list of all found seeds SeedVec seeds; // max number of matches for this read CountType max_num_matches; // set the read_length void set_rlen(CountType r); // get the size of the serialized object uint64_t serialize_size(); // serialize the object std::vector serialize(); // deserialize (read) data from a char vector uint64_t deserialize(char* d); // convert and return sequence of the read as string (without barcode) std::string getSequenceString(AlignmentSettings & settings); // convert and return sequence of the barcode std::string getBarcodeString(AlignmentSettings & settings); // append one nucleotide to sequenceStoreVector void appendNucleotideToSequenceStoreVector(char nuc); // extend the alignment by one basecall using reference database index void extend_alignment(char bc, KixRun* index, AlignmentSettings* settings); // disable this alignment void disable(AlignmentSettings & settings); // obtain start position of a seed according to SAM (leftmost) PositionType get_SAM_start_pos(USeed & sd, AlignmentSettings & settings); }; // END class ReadAlignment //-------------------------------------------------------------------// //------ Other helper functions -----------------------------------// //-------------------------------------------------------------------// int16_t MAPQ(const SeedVec &sv); #endif /* ALNREAD_H */ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/alnstream.cpp000066400000000000000000000635301301327033400236040ustar00rootroot00000000000000#include "alnstream.h" //-------------------------------------------------------------------// //------ The output Alignment Stream class ------------------------// //-------------------------------------------------------------------// // new output Alignment Stream class oAlnStream::oAlnStream(uint16_t ln, uint16_t tl, uint16_t cl, std::string rt, CountType rl, uint32_t nr, uint64_t bs, uint8_t fmt): lane(ln), tile(tl), cycle(cl), root(rt), rlen(rl), num_reads(nr), num_written(0), buffer(bs,0), buf_size(bs), buf_pos(0), format(fmt), ofile(NULL), ozfile(Z_NULL) {} // write function for lz4 compression uint64_t oAlnStream::lz4write(const char* source, uint64_t size) { // allocate buffer for the compressed data std::vector buf (LZ4_COMPRESSBOUND(size),0); // compress the data uint32_t compressed_size = LZ4_compress (source, buf.data(), size); if (!compressed_size) throw std::runtime_error("Error compressing data with LZ4."); // write the block size if ( !fwrite(&compressed_size, 1, sizeof(uint32_t), ofile) ) throw std::runtime_error("Error writing block size to file while compressing data with LZ4."); // write the data chunk if ( !fwrite(buf.data(), 1, compressed_size, ofile) ) throw std::runtime_error("Error writing data to file while compressing with LZ4."); return size; } uint64_t oAlnStream::open(std::string fname) { // open the new Alignment file switch (format) { case 0: case 2: ofile = fopen(fname.c_str(), "wb"); if (!ofile) { std::cerr << "Could not open file " << fname << " for writing." << std::endl; return 0; } break; case 1: ozfile = gzopen(fname.c_str(), "wb1"); //Don't compress too much, not enough bang for the buck if (ozfile == Z_NULL) { std::cerr << "Could not open file " << fname << " for writing." << std::endl; return 0; } break; default: throw std::invalid_argument("Output file format not recognized"); } // write the header: // calculate total size first unsigned long int total_size = 0; total_size += sizeof(uint16_t); // lane total_size += sizeof(uint16_t); // tile total_size += sizeof(CountType); // cycle // root directory name uint16_t root_size = root.size(); total_size += sizeof(uint16_t); total_size += root_size; // read length total_size += sizeof(CountType); // number of reads total_size += sizeof(uint32_t); // create the vector to store the data std::vector data (total_size); char* d = data.data(); // write the lane memcpy(d,&lane,sizeof(uint16_t)); d += sizeof(uint16_t); // write the tile memcpy(d,&tile,sizeof(uint16_t)); d += sizeof(uint16_t); // write the cycle memcpy(d,&cycle,sizeof(CountType)); d += sizeof(CountType); // root directory name memcpy(d,&root_size,sizeof(uint16_t)); d += sizeof(uint16_t); memcpy(d,root.c_str(),root_size); d += root_size; // write the read length memcpy(d,&rlen,sizeof(CountType)); d += sizeof(CountType); // write the number of reads memcpy(d,&num_reads,sizeof(uint32_t)); d += sizeof(int32_t); // write all data uint64_t written = 0; switch (format) { case 0: case 2: written = fwrite(data.data(), 1, data.size(), ofile); break; case 1: written = gzwrite(ozfile, data.data(), data.size()); break; } return written; } // writes a read alignment to the output Alignment file. // Buffering is handled internally uint64_t oAlnStream::write_alignment(ReadAlignment * al) { if ( (!ofile && (format == 0 || format == 2)) || (ozfile == Z_NULL && format == 1) ){ throw std::runtime_error("Could not write alignment to file. File handle not valid."); } if (num_written >= num_reads) { throw std::length_error("Could not write alignment to file. All alignments were already written."); } std::vector data = al->serialize(); uint64_t al_size = data.size(); // first, write the size of the serialized alignment (uint32_t = 4 bytes) if (buf_pos+sizeof(uint32_t) <= buf_size) { // directly copy if all 4 bytes have space in the buffer (should be almost always the case) memcpy(buffer.data()+buf_pos,&al_size,sizeof(uint32_t)); buf_pos += sizeof(uint32_t); } else { // copy the first bytes into temporary buffer to compose the alignment size std::vector temp (sizeof(uint32_t),0); memcpy(temp.data(),&al_size,sizeof(uint32_t)); uint64_t first_part = buf_size-buf_pos; memcpy(buffer.data()+buf_pos,temp.data(),first_part); // write out buffer uint64_t written = 0; switch (format) { case 0: written = fwrite(buffer.data(), 1, buffer.size(), ofile); break; case 1: written = gzwrite(ozfile, buffer.data(), buffer.size()); break; case 2: written = lz4write(buffer.data(), buffer.size()); break; } assert(written == buf_size); // copy remaining data memcpy(buffer.data(),temp.data()+first_part,sizeof(uint32_t)-first_part); buf_pos = sizeof(uint32_t)-first_part; } // finally, write the serialized data uint64_t copied = 0; while (copied < al_size) { uint64_t to_copy = std::min(al_size-copied,buf_size-buf_pos); memcpy(buffer.data()+buf_pos, data.data()+copied, to_copy); buf_pos += to_copy; copied += to_copy; // write buffer to disk if full if(buf_pos >= buf_size){ uint64_t written = 0; switch (format) { case 0: written = fwrite(buffer.data(), 1, buffer.size(), ofile); break; case 1: written = gzwrite(ozfile, buffer.data(), buffer.size()); break; case 2: written = lz4write(buffer.data(), buffer.size()); break; } assert(written == buf_size); buf_pos = 0; } } num_written++; return num_written; } bool oAlnStream::close() { if ( ((format == 0 || format == 2) && ofile) || (format == 1 && ozfile != Z_NULL) ) { // write remaining buffer content to file uint64_t written = 0; switch (format) { case 0: written = fwrite(buffer.data(), 1, buf_pos, ofile); break; case 1: written = gzwrite(ozfile, buffer.data(), buf_pos); break; case 2: written = lz4write(buffer.data(), buf_pos); break; } assert(written == buf_pos); buf_pos = 0; if (num_written == num_reads) { switch (format) { case 0: case 2: fclose(ofile); break; case 1: gzclose(ozfile); break; } return true; } else { std::cerr << "Error: Could not close output alignment file! "<< num_reads - num_written <<" alignments missing." << std::endl; return false; } } else { std::cerr << "Error: Could not close output alignment file! File handle not valid." << std::endl; return false; } } //-------------------------------------------------------------------// //------ The input Alignment Stream class -------------------------// //-------------------------------------------------------------------// // new Alignment Stream class iAlnStream::iAlnStream(uint64_t bs, uint8_t fmt): lane(0), tile(0), cycle(0), root(""), rlen(0), num_reads(0), num_loaded(0), buffer(bs,0), buf_size(bs), buf_pos(bs), format(fmt), ifile(NULL), izfile(Z_NULL) {} // read function for lz4 decompression, reads one block of data uint64_t iAlnStream::lz4read_block() { // get the size of the next block uint32_t compressed_size = 0; if ( !fread(&compressed_size,sizeof(uint32_t),1,ifile) ) return 0; // allocate buffer for the compressed data std::vector cbuf (compressed_size,0); // read the data if ( !fread(cbuf.data(),compressed_size,1,ifile) ) throw std::runtime_error("Malformed input file. Could not read next block."); // decompress the data int64_t r_size = LZ4_decompress_safe (cbuf.data(), buffer.data(), compressed_size, buffer.size()); if ( r_size < 0 ) throw std::runtime_error("Error while decompressing LZ4 compressed block."); // update the current buffer size buf_size = r_size; return (uint64_t)r_size; } uint64_t iAlnStream::open(std::string fname) { // open the new Alignment file switch (format) { case 0: case 2: ifile = fopen(fname.c_str(), "rb"); if (!ifile) { std::cerr << "Error opening file " << fname << " for reading." << std::endl; return 0; } break; case 1: izfile = gzopen(fname.c_str(), "rb"); if (izfile == Z_NULL) { std::cerr << "Error opening file " << fname << " for reading." << std::endl; return 0; } break; default: throw std::invalid_argument("Input file format not recognized."); } // load the header: uint64_t bytes = 0; uint16_t root_size; switch (format) { case 0: case 2: { // read the lane bytes += fread(&lane,sizeof(uint16_t),1,ifile); // read the tile bytes += fread(&tile,sizeof(uint16_t),1,ifile); // read the cycle bytes += fread(&cycle,sizeof(CountType),1,ifile); // root directory name size bytes += fread(&root_size,sizeof(uint16_t),1,ifile); // root name char * tmp = new char[root_size+1]; bytes += fread(tmp,1,root_size,ifile); tmp[root_size] = 0; // make the string null-terminated root = tmp; delete tmp; // read the read length bytes += fread(&rlen,sizeof(CountType),1,ifile); // read the number of reads bytes += fread(&num_reads,sizeof(uint32_t),1,ifile); break; } case 1: { // read the lane bytes += gzread(izfile,&lane,sizeof(uint16_t)); // read the tile bytes += gzread(izfile,&tile,sizeof(uint16_t)); // read the cycle bytes += gzread(izfile,&cycle,sizeof(CountType)); // root directory name size bytes += gzread(izfile,&root_size,sizeof(uint16_t)); // root name char * tmp = new char[root_size+1]; bytes += gzread(izfile,tmp,root_size); tmp[root_size] = 0; // make the string null-terminated root = tmp; delete tmp; // read the read length bytes += gzread(izfile,&rlen,sizeof(CountType)); // read the number of reads bytes += gzread(izfile,&num_reads,sizeof(uint32_t)); break; } } return bytes; } ReadAlignment* iAlnStream::get_alignment() { if ( (format==0 && !ifile) || (format==1 && izfile == Z_NULL) ){ throw std::runtime_error("Could not load alignment from file. File handle not valid."); } if (num_loaded >= num_reads) { throw std::length_error("Could not load alignment from file. All alignments were already loaded."); } // first, get the size of the serialized alignment (uint32_t = 4 bytes) uint32_t al_size = 0; if (buf_pos+sizeof(uint32_t) <= buf_size) { // directly copy if all 4 bytes are in the buffer (should be almost always the case) memcpy(&al_size,buffer.data()+buf_pos,sizeof(uint32_t)); buf_pos += sizeof(uint32_t); } else { // copy the first bytes into temporary buffer to compose the alignment size std::vector temp (sizeof(uint32_t),0); uint64_t first_part = buf_size-buf_pos; memcpy(temp.data(),buffer.data()+buf_pos,first_part); // load new buffer uint64_t loaded; switch (format) { case 0: loaded = fread(buffer.data(),1,buf_size,ifile); assert( (loaded == buf_size) || feof(ifile) ); break; case 1: loaded = gzread(izfile,buffer.data(),buf_size); assert( (loaded == buf_size) || gzeof(izfile) ); break; case 2: loaded = lz4read_block(); assert( loaded>0 || feof(ifile) ); break; } // copy remaining data and copy to variable memcpy(temp.data()+first_part,buffer.data(),sizeof(uint32_t)-first_part); buf_pos = sizeof(uint32_t)-first_part; memcpy(&al_size,temp.data(),sizeof(uint32_t)); } // then, copy the content to the data vector std::vector data(al_size,0); uint64_t copied = 0; while (copied < al_size) { uint64_t to_copy = std::min(al_size-copied,buf_size-buf_pos); memcpy(data.data()+copied, buffer.data()+buf_pos, to_copy); buf_pos += to_copy; copied += to_copy; // read new buffer from disk if necessary if(buf_pos >= buf_size){ uint64_t loaded; switch (format) { case 0: loaded = fread(buffer.data(),1,buf_size,ifile); assert( (loaded == buf_size) || feof(ifile) ); break; case 1: loaded = gzread(izfile,buffer.data(),buf_size); assert( (loaded == buf_size) || gzeof(izfile) ); break; case 2: loaded = lz4read_block(); assert( loaded>0 || feof(ifile) ); break; } buf_pos = 0; } } // finally, deserialize the alignment ReadAlignment* ra = new ReadAlignment(); ra->set_rlen(rlen); ra->deserialize(data.data()); num_loaded++; return ra; } bool iAlnStream::close() { //if (ifile) { if ( ((format==0 || format==2) && ifile) || (format==1 && izfile != Z_NULL)) { if (num_loaded == num_reads) { switch (format) { case 0: case 2: fclose(ifile); break; case 1: gzclose(izfile); break; } return true; } else { std::cerr << "Error: Could not close alignment file! "<< num_reads - num_loaded <<" alignments missing." << std::endl; return false; } } else { throw std::runtime_error("Could not close alignment file. File handle not valid."); } } //-------------------------------------------------------------------// //------ The StreamedAlignment class ------------------------------// //-------------------------------------------------------------------// StreamedAlignment& StreamedAlignment::operator=(const StreamedAlignment& other) { if(&other == this) return *this; lane = other.lane; tile = other.tile; root = other.root; rlen = other.rlen; return *this; } std::string StreamedAlignment::get_bcl_file(uint16_t cycle) { std::ostringstream path_stream; path_stream << root << "/L00" << lane << "/C" << cycle << ".1/s_"<< lane <<"_" << tile << ".bcl"; return path_stream.str(); } // get the path to the alignment file. The alignment file is located in // /L00/s__..align // if base == "": base = root std::string StreamedAlignment::get_alignment_file(uint16_t cycle, std::string base){ if (base == "") { base = root; } std::ostringstream path_stream; path_stream << base << "/L00" << lane << "/s_"<< lane << "_" << tile << "." << cycle << ".align"; return path_stream.str(); } std::string StreamedAlignment::get_filter_file() { std::ostringstream path_stream; path_stream << root << "/L00" << lane << "/s_"<< lane << "_" << tile << ".filter"; return path_stream.str(); } // create directories required to store the alignment files (only if not stored in root) void StreamedAlignment::create_directories(AlignmentSettings* settings) { std::ostringstream path_stream; if (settings->temp_dir == "") { path_stream << root; } else { path_stream << settings->temp_dir; } path_stream << "/L00" << lane; boost::filesystem::create_directories(path_stream.str()); std::ostringstream sam_stream; sam_stream << settings->out_dir; sam_stream << "/L00" << lane; boost::filesystem::create_directories(sam_stream.str()); } // initialize empty alignment. Creates alignment files for a virtual Cycle 0 void StreamedAlignment::init_alignment(AlignmentSettings* settings) { std::string out_fname = get_alignment_file(0, settings->temp_dir); // get the number of reads in this tile by looking in the first bcl file std::string first_cycle = get_bcl_file(1); // extract the number of reads uint32_t num_reads = num_reads_from_bcl(first_cycle); // open output alignment stream oAlnStream output (lane, tile, 0, root, rlen, num_reads, settings->block_size, settings->compression_format); output.open(out_fname); // write empty read alignments for each read for (uint32_t i = 0; i < num_reads; ++i) { ReadAlignment * ra = new ReadAlignment(); ra->set_rlen(rlen); output.write_alignment(ra); delete ra; } if(!output.close()) { std::cerr << "Error: Could not create initial alignment file." << std::endl; } } // extend an existing alignment from cycle to . returns the number of seeds uint64_t StreamedAlignment::extend_alignment(uint16_t cycle, KixRun* index, AlignmentSettings* settings) { // 1. Open the input file //----------------------- std::string in_fname = get_alignment_file(cycle-1, settings->temp_dir); std::string bcl_fname = get_bcl_file(cycle); std::string filter_fname = get_filter_file(); iAlnStream input ( settings->block_size, settings->compression_format ); input.open(in_fname); assert(input.get_cycle() == cycle-1); assert(input.get_lane() == lane); assert(input.get_tile() == tile); assert(input.get_root() == root); assert(input.get_rlen() == rlen); uint32_t num_reads = input.get_num_reads(); // 2. Open output stream //---------------------------------------------------------- std::string out_fname = get_alignment_file(cycle, settings->temp_dir); oAlnStream output (lane, tile, cycle, root, rlen, num_reads, settings->block_size, settings->compression_format); output.open(out_fname); // 3. Read the full BCL file (this is not too much) //------------------------------------------------- BclParser basecalls; basecalls.open(bcl_fname); // extract the number of reads from the BCL file uint32_t num_base_calls = basecalls.size(); assert(num_base_calls == num_reads); // 4. Load the filter flags if filter file is available // ---------------------------------------------------- FilterParser filters; if (file_exists(filter_fname)) { filters.open(filter_fname); // extract the number of reads from the filter file uint32_t num_reads_filter = filters.size(); if (num_reads != num_reads_filter){ std::string msg = std::string("Number of reads in filter file (") + std::to_string(num_reads_filter) + ") does not match the number of reads in the BCL file (" + std::to_string(num_reads) + ")."; throw std::length_error(msg.c_str()); } } // 5. Extend alignments 1 by 1 //------------------------------------------------- uint64_t num_seeds = 0; for (uint64_t i = 0; i < num_reads; ++i) { ReadAlignment* ra = input.get_alignment(); if (filters.size() > 0 && filters.has_next()) { // filter file was found -> apply filter if(filters.next()) { ra->extend_alignment(basecalls.next(), index, settings); num_seeds += ra->seeds.size(); } else { basecalls.next(); ra->disable(*settings); } } // filter file was not found -> treat every alignment as valid else { ra->extend_alignment(basecalls.next(), index, settings); num_seeds += ra->seeds.size(); } output.write_alignment(ra); delete ra; } // 6. Close files //------------------------------------------------- if (!(input.close() && output.close())) { std::cerr << "Could not finish alignment!" << std::endl; } // 7. Delete old alignment file, if requested //------------------------------------------- if ( ! settings->keep_aln_files ) { std::remove(in_fname.c_str()); } return num_seeds; } //-------------------------------------------------------------------// //------ Streamed SAM generation -----------------------------------// //-------------------------------------------------------------------// uint64_t alignments_to_sam(uint16_t ln, uint16_t tl, std::string rt, CountType rl, KixRun* index, AlignmentSettings* settings) { // set the file names std::string temp; if (settings->temp_dir == "") temp = rt; else temp = settings->temp_dir; std::string sam_dir = settings->out_dir; std::string filter_fname = filter_name(rt, ln, tl); std::string alignment_fname = alignment_name(temp, ln, tl, rl); std::string sam_fname = sam_tile_name(sam_dir, ln, tl, settings->write_bam); // check if files exist if ( !file_exists(alignment_fname) ) throw std::runtime_error(std::string("Could not create SAM file. Alignment file not found: ")+ alignment_fname); if ( !file_exists(filter_fname) ) std::cerr << "Could not find .filter file: " << filter_fname << std::endl; // open the alignment file iAlnStream input ( settings->block_size, settings->compression_format ); input.open(alignment_fname); uint64_t num_reads = input.get_num_reads(); // open the filter file, if applicable FilterParser filters; if (file_exists(filter_fname)) { filters.open(filter_fname); // extract the number of reads from the filter file uint32_t num_reads_filter = filters.size(); if (num_reads != num_reads_filter){ std::string msg = std::string("Number of reads in filter file (") + std::to_string(num_reads_filter) + ") does not match the number of reads in the BCL file (" + std::to_string(num_reads) + ")."; throw std::length_error(msg.c_str()); } } ///////////////////////////////////////////// // generate sam file // set BamFileOut object seqan::CharString samFileName = sam_fname; seqan::BamFileOut samFileOut(seqan::toCString(samFileName)); seqan::StringSet referenceNames; for (unsigned i=0; iseq_names); i++) { seqan::appendValue(referenceNames, index->seq_names[i]); } seqan::NameStoreCache > referenceNamesCache(referenceNames); seqan::BamIOContext > bamIOContext(referenceNames, referenceNamesCache); samFileOut.context = bamIOContext; ///////////////// // set SAM header seqan::BamHeaderRecord headerRecord; // @HD header. seqan::clear(headerRecord); headerRecord.type = seqan::BAM_HEADER_FIRST; seqan::resize(headerRecord.tags, 2); headerRecord.tags[0].i1 = "VN"; headerRecord.tags[0].i2 = "1.5"; headerRecord.tags[1].i1 = "GO"; headerRecord.tags[1].i2 = "query"; seqan::writeHeader(samFileOut, headerRecord); // @SQ header. std::stringstream ss; for ( uint64_t i = 0; i < index->seq_names.size(); i++ ) { ss.str(std::string()); // clear string stream seqan::clear(headerRecord); headerRecord.type = seqan::BAM_HEADER_REFERENCE; seqan::resize(headerRecord.tags, 2); headerRecord.tags[0].i1 = "SN"; headerRecord.tags[0].i2 = index->seq_names[i]; headerRecord.tags[1].i1 = "LN"; ss << index->seq_lengths[i]; headerRecord.tags[1].i2 = ss.str(); seqan::writeHeader(samFileOut, headerRecord); } // @PG header. seqan::clear(headerRecord); headerRecord.type = seqan::BAM_HEADER_PROGRAM; seqan::resize(headerRecord.tags, 3); headerRecord.tags[0].i1 = "ID"; headerRecord.tags[0].i2 = "hilive"; headerRecord.tags[1].i1 = "PN"; headerRecord.tags[1].i2 = "HiLive"; headerRecord.tags[2].i1 = "VN"; ss.str(std::string()); ss << HiLive_VERSION_MAJOR << "." << HiLive_VERSION_MINOR; headerRecord.tags[2].i2 = ss.str(); seqan::writeHeader(samFileOut, headerRecord); ///////////////// // prepare sam entries uint64_t num_alignments = 0; for (uint64_t i = 0; i < num_reads; ++i) { ReadAlignment * ra = input.get_alignment(); // if either: filter file is open and all filter flags were loaded and the filter flag is > 0 // or: the filter file is not available // then proceed else skip if (!((filters.size()>0 && filters.next()) || filters.size() == 0)) continue; num_alignments += ra->seeds.size(); // Read name format :::::: //readname << ":::" << ln << ":" << tl << ":::" << i; std::stringstream readname; readname << "lane." << ln << "|tile." << tl << "|read." << i; ///////////////// // set sam entries seqan::BamAlignmentRecord record; for (SeedVecIt it = ra->seeds.begin(); it != ra->seeds.end(); ++it) { seqan::clear(record); record.qName = readname.str(); record.rID = (*it)->gid; record.beginPos = ra->get_SAM_start_pos(*it, *settings)-1; // seqan expects 0-based positions, but in HiLive we use 1-based if (record.beginPos < 0) continue; record.cigar = (*it)->returnSeqanCigarString(); // flag and seq record.flag = 0; record.seq = ra->getSequenceString(*settings); if ((*it)->start_pos < 0) { // if read matched reverse complementary seqan::reverseComplement(record.seq); record.flag |= 16; } if (it != ra->seeds.begin()) { // if current seed is secondary alignment record.flag |= 256; seqan::clear(record.seq); record.qual = "*"; } // check if cigar string sums up to read length unsigned cigarElemSum = 0; unsigned deletionSum = 0; for (seqan::Iterator > >::Type elem = seqan::begin(record.cigar); elem != end(record.cigar); ++elem) { if ((elem->operation == 'M') || (elem->operation == 'I') || (elem->operation == 'S') || (elem->operation == '=') || (elem->operation == 'X')) cigarElemSum += elem->count; if (elem->operation == 'D') deletionSum += elem->count; } if (cigarElemSum != settings->seqlen) { std::cerr << "WARNING: Excluded an alignment of read " << record.qName << " at position " << ra->get_SAM_start_pos(*it, *settings) << " because its cigar vector had length " << cigarElemSum << std::endl; continue; } // tags seqan::BamTagsDict dict; seqan::appendTagValue(dict, "AS", (*it)->num_matches); if (settings->seqlen < settings->rlen) // if demultiplexing is on seqan::appendTagValue(dict, "BC", ra->getBarcodeString(*settings)); seqan::appendTagValue(dict, "NM", deletionSum + settings->seqlen - (*it)->num_matches); record.tags = seqan::host(dict); // write record to disk seqan::writeRecord(samFileOut, record); } } std::ofstream statsfile; statsfile.open(sam_fname+".stats"); statsfile << "Number of reads\t" << num_reads << std::endl; statsfile << "Number of alignments\t" << num_alignments << std::endl; statsfile.close(); return 0; } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/alnstream.h000066400000000000000000000121241301327033400232420ustar00rootroot00000000000000#ifndef ALNSTREAM_H #define ALNSTREAM_H #include "headers.h" #include "definitions.h" #include "kindex.h" #include "tools.h" #include "alnread.h" #include "illumina_parsers.h" #include #include #include // Output alignment stream: write alignments to file one by one class oAlnStream { // dataset information for the header uint16_t lane; uint16_t tile; uint16_t cycle; std::string root; CountType rlen; uint32_t num_reads; // number of reads written to file uint32_t num_written; // data buffer (don't write everything at once) std::vector buffer; // size of the buffer uint64_t buf_size; // current position in the buffer uint64_t buf_pos; // output file format // 0: no compression // 1: zlib compression (level 1) // 11: lz4 compression (level 1) uint8_t format; // file handles FILE* ofile; gzFile ozfile; // write function for lz4 compression uint64_t lz4write(const char* buf, uint64_t size); public: // constructor initializes all member variables oAlnStream(uint16_t ln, uint16_t tl, uint16_t cl, std::string rt, CountType rl, uint32_t nr, uint64_t bs, uint8_t fmt); // open Alignment stream file and write header uint64_t open(std::string fname); // writes a read alignment to the output Alignment file. // Buffering is handled internally uint64_t write_alignment(ReadAlignment * al); // checks if the correct number of alignments was written and closes the Alignment file bool close(); }; // Input alignment stream: loads read alignments from a file one by one class iAlnStream { // dataset information for the header uint16_t lane; uint16_t tile; uint16_t cycle; std::string root; CountType rlen; uint32_t num_reads; // number of reads loaded from file uint32_t num_loaded; // data buffer (read blocks of data) std::vector buffer; // size of the buffer uint64_t buf_size; // current position in the buffer uint64_t buf_pos; // output file format // 0: no compression // 1: zlib compression (level 1) // 11: lz4 compression (level 1) uint8_t format; // file pointer FILE* ifile; gzFile izfile; // read function for LZ4 compression. Reads one block of data to buffer uint64_t lz4read_block(); public: // constructor initializes only block size and file format iAlnStream(uint64_t bs, uint8_t fmt); // open Alignment stream file and load header uint64_t open(std::string fname); // loads a read alignment from the input Alignment file. // Buffering is handled internally ReadAlignment* get_alignment(); // checks if the correct number of alignments was loaded and closes the Alignment file bool close(); // get dataset information inline uint16_t get_lane() {return lane;}; inline uint16_t get_tile() {return tile;}; inline uint16_t get_cycle() {return cycle;}; inline std::string get_root() {return root;}; inline CountType get_rlen() {return rlen;}; inline uint32_t get_num_reads() {return num_reads;}; inline uint32_t get_num_loaded() {return num_loaded;}; }; //-------------------------------------------------------------------// //------ The StreamedAlignment class ------------------------------// //-------------------------------------------------------------------// class StreamedAlignment { // dataset information uint16_t lane; uint16_t tile; std::string root; // the BaseCalls directory CountType rlen; // fetch the next read from the input stream ReadAlignment get_next_read(); // write an alignment to the output stream uint64_t write_alignment(ReadAlignment& ral); // get the path to the bcl file of a given cycle std::string get_bcl_file(uint16_t cycle); // get the path to the alignment file. The alignment file is located in // /L00/s__..align // if base == "": base = root std::string get_alignment_file(uint16_t cycle, std::string base = ""); // get the path to the filter file. The illumina filter information is located in // /L00/s__.filter std::string get_filter_file(); public: StreamedAlignment(uint16_t ln, uint16_t tl, std::string rt, CountType rl): lane(ln), tile(tl), root(rt), rlen(rl) {}; StreamedAlignment& operator=(const StreamedAlignment& other); // create directories required to store the alignment files (only if not stored in root) void create_directories(AlignmentSettings* settings); // initialize empty alignment. Creates files for a virtual Cycle 0 void init_alignment(AlignmentSettings* settings); // extend an existing alignment from cycle to uint64_t extend_alignment(uint16_t cycle, KixRun* index, AlignmentSettings* settings); }; /* END class StreamedAlignment */ //-------------------------------------------------------------------// //------ Streamed SAM generation -----------------------------------// //-------------------------------------------------------------------// uint64_t alignments_to_sam(uint16_t ln, uint16_t tl, std::string rt, CountType rl, KixRun* index, AlignmentSettings* settings); #endif /* ALNSTREAM_H */ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/argument_parser.cpp000066400000000000000000000262501301327033400250120ustar00rootroot00000000000000#include "argument_parser.h" namespace po = boost::program_options; int parseCommandLineArguments(AlignmentSettings & settings, std::string license, int argc, char const ** argv) { po::options_description general("General"); general.add_options() ("help,h", "Print this help message and exit") ("license", "Print licensing information and exit"); po::options_description parameters("Parameters"); parameters.add_options() ("BC_DIR", po::value(&settings.root)->required(), "Illumina BaseCalls directory") ("INDEX", po::value(&settings.index_fname)->required(), "Path to k-mer index") ("CYCLES", po::value(&settings.rlen)->required(), "Number of cycles") ("OUTDIR", po::value(&settings.out_dir), "Directory to store sam files in [Default: temporary or BaseCalls directory"); po::options_description io_settings("IO settings"); io_settings.add_options() ("temp", po::value(&settings.temp_dir)->default_value(""), "Temporary directory for the alignment files [Default: use BaseCalls directory]") //("bam,B", po::bool_switch(&settings.write_bam)->default_value(false), "Create BAM files instead of SAM files [Default: false]") ("keep-files,k", po::bool_switch(&settings.keep_aln_files)->default_value(false), "Keep intermediate alignment files [Default: false]") ("lanes,l", po::value< std::vector >()->multitoken()->composing(), "Select lane [Default: all lanes]") ("tiles,t", po::value< std::vector >()->multitoken()->composing(), "Select tile numbers [Default: all tiles]") ("barcodes,b", po::value< std::vector >()->multitoken()->composing(), "Enumerate barcodes (must have same length) for demultiplexing, i.e. -b AGGATC -b CCCTTT [Default: no demultiplexing]"); po::options_description alignment("Alignment settings"); alignment.add_options() ("min-errors,e", po::value(&settings.min_errors)->default_value(2), "Number of errors tolerated in read alignment [Default: 2]") ("all-best-hit,H", po::bool_switch()->default_value(false), "Report all of the best alignments for each read") ("any-best-hit", po::bool_switch(), "Report one of the best alignments for each read (default)") ("all-best-n-scores,N", po::value(&settings.best_n), "Report all alignments of the N best alignment scores for each read") ("all-hits,A", po::bool_switch()->default_value(false), "Report all valid alignments for each read") ("disable-ohw-filter", po::bool_switch(&settings.discard_ohw)->default_value(true), "disable the One-Hit Wonder filter [Default: false]") ("start-ohw", po::value(&settings.start_ohw)->default_value(K_HiLive+5), "First cycle to apply One-Hit Wonder filter [Default: K+5]") ("window,w", po::value(&settings.window)->default_value(5), "Set the window size to search for alignment extension, i.e. maximum total insertion/deletion size [Default: 5]") ("min-quality", po::value(&settings.min_qual)->default_value(1), "Minimum allowed basecall quality [Default: 1]"); po::options_description technical("Technical settings"); technical.add_options() ("block-size", po::value()->default_value(64*1024*1024), "Block size for the alignment input/output stream in Bytes. Use -K or -M to specify in Kilobytes or Megabytes") (",K", po::bool_switch()->default_value(false), "Interpret the block-size argument as Kilobytes instead of Bytes") (",M", po::bool_switch()->default_value(false), "Interpret the block-size argument as Megabytes instead of Bytes") ("compression,c", po::value(&settings.compression_format)->default_value(2), "Compress alignment files. 0: no compression (default) 1: Deflate (smaller) 2: LZ4 (faster)") ("num-threads,n", po::value(), "Number of threads to spawn [Default: all available]"); po::options_description cmdline_options; cmdline_options.add(general).add(parameters).add(io_settings).add(alignment).add(technical); po::options_description visible_options; visible_options.add(general).add(io_settings).add(alignment).add(technical); std::stringstream help_message; help_message << "HiLive v"<< HiLive_VERSION_MAJOR << "." << HiLive_VERSION_MINOR << " - Realtime Alignment of Illumina Reads" << std::endl; help_message << "Copyright (c) 2015, Martin S. Lindner" << std::endl; help_message << "HiLive is open-source software. Check with --license for details." << std::endl << std::endl; help_message << "Fixed k-mer size: " << K_HiLive << std::endl << std::endl; help_message << "Usage: " << std::string(argv[0]) << " [options] BC_DIR INDEX CYCLES OUTDIR" << std::endl; help_message << " BC_DIR Illumina BaseCalls directory of the sequencing run to analyze" << std::endl; help_message << " INDEX Path to k-mer index file (*.kix)" << std::endl; help_message << " CYCLES Total number of cycles for read 1" << std::endl; help_message << " OUTDIR Directory to store sam files in" << std::endl; help_message << visible_options << std::endl; po::positional_options_description p; p.add("BC_DIR", 1); p.add("INDEX", 1); p.add("CYCLES", 1); p.add("OUTDIR", 1); // parse the arguments po::variables_map vm; try { // parse arguments po::store(po::command_line_parser(argc, argv).options(cmdline_options).positional(p).run(), vm); // first check if -h or --help was called if (vm.count("help")) { std::cout << help_message.str(); return 1; } // first check if --license was called if (vm.count("license")) { std::cout << license << std::endl; return 1; } // then check arguments po::notify(vm); } catch ( boost::program_options::required_option& e ) { std::cerr << "Missing Parameter: " << e.what() << std::endl << std::endl; std::cout << help_message.str(); return -1; } catch( boost::program_options::error& e) { std::cerr << "Error while parsing command line options: " << e.what() << std::endl << std::endl; std::cout << help_message.str(); return -1; } if (vm.count("OUTDIR")) settings.out_dir = vm["OUTDIR"].as(); else { if (settings.temp_dir == "") settings.out_dir = settings.root; else settings.out_dir = settings.temp_dir; } if (vm.count("lanes")) settings.lanes = vm["lanes"].as< std::vector >(); else settings.lanes = all_lanes(); if (vm.count("tiles")) settings.tiles = vm["tiles"].as< std::vector >(); else settings.tiles = all_tiles(); settings.barcodeVector.clear(); settings.seqlen = settings.rlen; if (vm.count("barcodes")) { settings.barcodeVector = vm["barcodes"].as< std::vector >(); settings.seqlen = settings.rlen - settings.barcodeVector[0].size(); } if (vm["all-hits"].as()) { // all hits: disable other modes settings.any_best_hit_mode = false; settings.all_best_hit_mode = false; settings.all_best_n_scores_mode = false; } else if (vm["all-best-hit"].as()) { // enable all-best-hit mode and disable others settings.any_best_hit_mode = false; settings.all_best_hit_mode = true; settings.all_best_n_scores_mode = false; } else if (vm.count("all-best-n-scores")) { // enable any-best-n mode and get parameter settings.any_best_hit_mode = false; settings.all_best_hit_mode = false; settings.all_best_n_scores_mode = true; settings.best_n = vm["all-best-n-scores"].as(); } else { // the default behaviour // enable any-best-hit mode and disable others settings.any_best_hit_mode = true; settings.all_best_hit_mode = false; settings.all_best_n_scores_mode = false; } if (vm["-M"].as()) settings.block_size = vm["block-size"].as()*1024*1024; else if (vm["-K"].as()) settings.block_size = vm["block-size"].as()*1024; else settings.block_size = vm["block-size"].as(); if (vm.count("num-threads")) settings.num_threads = vm["num-threads"].as(); else { // the default case, meaning as much as physically useful uint32_t n_cpu = std::thread::hardware_concurrency(); if (n_cpu > 1) settings.num_threads = n_cpu; else settings.num_threads = 1; } // check paths and file names if (!file_exists(settings.index_fname)){ std::cerr << "Input error: Could not find k-mer index file " << settings.index_fname << std::endl; return -1; } std::size_t found = settings.root.find("BaseCalls"); if (!(found != std::string::npos && found >= settings.root.size()-10)) { std::cerr << "Warning: BaseCalls directory seems to be invalid: " << settings.root << std::endl; } if (!is_directory(settings.root)){ std::cerr << "Input error: Could not find BaseCalls directory " << settings.root << std::endl; return -1; } for ( uint16_t ln : settings.lanes ) { std::string ln_dir = settings.root; if ( ln < 10 ) ln_dir += "/L00"; else if ( ln < 100 ) ln_dir += "/L0"; else ln_dir += "/L"; ln_dir += std::to_string(ln); if (!is_directory(ln_dir)){ std::cerr << "Input error: Could not find location of Lane " << ln << ": " << ln_dir << std::endl; return -1; } } // Report the basic settings std::cout << "Running HiLive with " << settings.num_threads << " thread(s)." << std::endl; std::cout << "BaseCalls directory: " << settings.root << std::endl; if (settings.temp_dir != "") { std::cout << "Temporary directory: " << settings.temp_dir << std::endl; } //if (!settings.write_bam) std::cout << "SAM output directory: " << settings.out_dir << std::endl; //else //std::cout << "BAM output directory: " << settings.out_dir << std::endl; std::cout << "Lanes: "; for ( uint16_t ln : settings.lanes ) std::cout << ln << " "; std::cout << std::endl; std::cout << "K-mer index: " << settings.index_fname << std::endl; std::cout << "Read length: " << settings.rlen << std::endl; std::cout << "Mapping error: " << settings.min_errors << std::endl; if (settings.any_best_hit_mode) std::cout << "Mapping mode: Any-Best-Hit-Mode" << std::endl; else if (settings.all_best_hit_mode) std::cout << "Mapping mode: All-Best-Hit-Mode" << std::endl; else if (settings.all_best_n_scores_mode) std::cout << "Mapping mode: All-Best-N-Scores-Mode with N=" << settings.best_n << std::endl; else std::cout << "Mapping mode: All-Hits-Mode" << std::endl; std::cout << std::endl; return 0; } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/argument_parser.h000066400000000000000000000004311301327033400244500ustar00rootroot00000000000000#include #include #include "headers.h" #include "definitions.h" #include "kindex.h" #include "alnstream.h" #include "parallel.h" int parseCommandLineArguments(AlignmentSettings & settings, std::string license, int argc, char const ** argv); HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/config.h.in000066400000000000000000000002671301327033400231330ustar00rootroot00000000000000// the k-mer length #define K_HiLive @HiLive_K@ // the HiLive Version Number #define HiLive_VERSION_MAJOR @HiLive_VERSION_MAJOR@ #define HiLive_VERSION_MINOR @HiLive_VERSION_MINOR@ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/definitions.h000066400000000000000000000121251301327033400235700ustar00rootroot00000000000000#ifndef DEFINITIONS_H #define DEFINITIONS_H #include "headers.h" // bit representation of A/C/G/T. #define twobit_repr(ch) ((toupper(ch)) == 'A' ? 0LL : \ (toupper(ch)) == 'C' ? 1LL : \ (toupper(ch)) == 'G' ? 2LL : 3LL) // complement bit representation of A/C/G/T. #define twobit_comp(ch) ((toupper(ch)) == 'A' ? 3LL : \ (toupper(ch)) == 'C' ? 2LL : \ (toupper(ch)) == 'G' ? 1LL : 0LL) // bit representation to character #define revtwobit_repr(n) ((n) == 0 ? 'A' : \ (n) == 1 ? 'C' : \ (n) == 2 ? 'G' : 'T') // Allowed characters in sequence const std::string seq_chars = "ACGTacgt"; // largest number we're going to hash into. (8 bytes/64 bits/32 nt) // probably 32 bit/16 nt are enough here typedef uint64_t HashIntoType; // construct a mask to truncate a binary representation of a k-mer to length K const HashIntoType MASK = HashIntoType(pow(4,K_HiLive))-1; // identifiers for genome sequences typedef uint32_t GenomeIdType; const GenomeIdType TRIMMED = std::numeric_limits::max(); // list of genome identifiers typedef std::vector GenomeIdListType; // list of strings typedef std::vector StringListType; // position in a genome typedef int32_t PositionType; // pair of genome ID and position struct GenomePosType { GenomeIdType gid; PositionType pos; GenomePosType()=default; GenomePosType(GenomeIdType g, PositionType p): gid(g), pos(p) {}; }; //typedef std::tuple GenomePosType; // size of a pair of genome ID and position const uint64_t GenomePos_size = sizeof(GenomeIdType) + sizeof(PositionType); // vector of ID:position pairs typedef std::vector GenomePosListType; // iterator on GenomePosList typedef GenomePosListType::iterator GenomePosListIt; // the k-mer index array const HashIntoType n_kmer = pow(4,K_HiLive); typedef std::array KmerIndexType; // small counters typedef uint16_t CountType; // difference between k-mer position in the read and matching position in the reference typedef int16_t DiffType; // define a mismatch as max(DiffType) const DiffType NO_MATCH = std::numeric_limits::max(); // define a trimmed position as max(DiffType)-1 const DiffType TRIMMED_MATCH = std::numeric_limits::max()-1; // one element in Cigar vector containing match/mismatch information about consecutive k-mers struct CigarElement { CountType length; DiffType offset; CigarElement (CountType l, DiffType o): length(l), offset(o) {}; CigarElement (): length(0), offset(NO_MATCH) {}; }; // CigarVector containing CIGAR string like information about the alignments typedef std::list CigarVector; // all user parameters are stored in the alignment settings struct AlignmentSettings { // HARD CODED: kmer gap structure (this is not used anywhere) //std::string kmer_structure = "11111110111110111"; std::string kmer_structure = "111111111111111"; // HARD CODED: kmer gap positions (one-based) //std::vector kmer_gaps = {8, 14}; std::vector kmer_gaps; // HARD CODED: kmer span (kmer weight is K_HiLive) //unsigned kmer_span = K_HiLive+2; unsigned kmer_span = K_HiLive; // PARAMETER: Base Call quality cutoff, treat BC with quality < bc_cutoff as miscall CountType min_qual; // PARAMETER: max. insert/deletion size DiffType window; // PARAMETER: minimum number of errors allowed in alignment CountType min_errors; // SWITCH: discard One-hit-wonders bool discard_ohw; // PARAMETER: first cycle to discard one-hit-wonders CountType start_ohw; // SWITCH: Best-Hit-Mode bool any_best_hit_mode; // SWITCH: Best-Hit-Mode bool all_best_hit_mode; // SWITCH: Best-N-Mode bool all_best_n_scores_mode; // PARAMETER: Best-N-Mode::N CountType best_n; // PARAMETER: temporary directory for the streamed alignment std::string temp_dir; // SWITCH: write sam/bam output or not bool write_bam=false; // SWITCH: Keep the old alignment files of previous cycles bool keep_aln_files; // PARAMETER: Memory block size for the input and output buffer in the streamed alignment uint64_t block_size; // PARAMETER: Compression format for alignment files uint8_t compression_format; // PARAMETER: list of lanes to process std::vector lanes; // PARAMETER: list of tiles to process std::vector tiles; // PARAMETER: root directory of hilive run std::string root; // PARAMETER: path to the index file std::string index_fname; // PARAMETER: read length of all reads (including barcodes) CountType rlen; // PARAMETER: length of the sequence of all reads (excluding barcodes) CountType seqlen; // PARAMETER: vector containing all barcodes of the reads which should be outputted std::vector barcodeVector; // PARAMETER: directory in which to create the output directory structure std::string out_dir; // PARAMETER: number of threads to use CountType num_threads; }; #endif /* DEFINITIONS_H */ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/headers.h000066400000000000000000000007551301327033400226760ustar00rootroot00000000000000#include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/illumina_parsers.cpp000066400000000000000000000032511301327033400251610ustar00rootroot00000000000000#include "illumina_parsers.h" // Constructor takes filename and directly loads the whole file uint64_t BclParser::open (std::string fname) { // read the whole file as a chunk data = read_binary_file(fname); // extract the number of reads memcpy(&num_reads,data.data(),4); // set the position pointer to the beginning of the data block position = 4; return data.size(); } // Get the next base call char BclParser::next() { if ( position < data.size() ) { position++; return *(data.data()+position-1); } else { throw std::runtime_error("Error reading BCL file: requested position is beyond EOF." ); } } // Check if there are base calls left bool BclParser::has_next() { return (position < data.size()); } // Returns the total number of base calls in the file uint32_t BclParser::size() { return num_reads; } // Constructor takes filename and directly loads the whole file uint64_t FilterParser::open (std::string fname) { // read the whole file as a chunk data = read_binary_file(fname); // extract the number of reads memcpy(&num_reads,data.data()+8,4); // set the position pointer to the beginning of the data block position = 12; return data.size(); } // Get the next filter flag bool FilterParser::next() { if ( position < data.size() ) { position++; return (*(data.data()+position-1) > 0); } else { throw std::runtime_error("Error reading filter file: requested position is beyond EOF." ); } } // Check if there are filter flags left bool FilterParser::has_next() { return (position < data.size()); } // Returns the total number of filter flags in the file uint32_t FilterParser::size() { return num_reads; } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/illumina_parsers.h000066400000000000000000000022171301327033400246270ustar00rootroot00000000000000#ifndef ILLUMINA_PARSERS_H #define ILLUMINA_PARSERS_H #include "headers.h" #include "definitions.h" #include "tools.h" // BCL file parser class BclParser { // storage for the raw binary data std::vector data; // current position in data uint32_t position; // number of reads in this bcl file uint32_t num_reads; public: // open file and directly load all data uint64_t open(std::string fname); // Get the next base call char next(); // Check if there are base calls left bool has_next(); // Returns the total number of base calls in the file uint32_t size(); }; // filter file parser class FilterParser { // storage for the raw binary data std::vector data; // current position in data uint32_t position; // number of reads in this filter file uint32_t num_reads; public: // open file and directly load all data uint64_t open(std::string fname); // Get the next filter flag bool next(); // Check if there are filter flags left bool has_next(); // Returns the total number of filter flags in the file uint32_t size(); }; // clocs file parser #endif /* ILLUMINA_PARSERS_H */ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/kindex.cpp000066400000000000000000000334101301327033400230720ustar00rootroot00000000000000#include "kindex.h" int KixBuild::add_fasta(const std::string &fname, AlignmentSettings & settings, bool convert_spaces, bool trim_ids) { GenomeIdListType added_ids; return add_fasta(fname, added_ids, settings, convert_spaces, trim_ids); } int KixBuild::add_fasta(const std::string &fname, GenomeIdListType &ids, AlignmentSettings & settings, bool convert_spaces, bool trim_ids) { std::ios::sync_with_stdio(false); std::ifstream::sync_with_stdio(false); std::ifstream infile (fname.c_str()); assert(infile.is_open()); std::string line; GenomeIdType seq_id = 0; std::string seq_name; GenomeIdListType added_ids; PositionType sequencePosition; bool startNewSequence = false; std::string tailingKmer; while(getline(infile, line)) { if (line.length() == 0) {continue;}; if (line[line.length()-1] == '\r'){ // check for Windows newline characters (just in case the fasta comes from Windows --> thanks Simon) line.erase(line.length()-1); } if (line[0] == '>') { // header line // initialize a new sequence num_seq += 1; seq_id = num_seq - 1; if (convert_spaces) std::replace( line.begin(), line.end(), ' ', '_'); if (trim_ids) seq_name = line.substr(1,line.find(' ')-1); else seq_name = line.substr(1,line.length()-1); added_ids.push_back(seq_id); seq_names.push_back(seq_name); seq_lengths.push_back(0); // gets later corrected to settings.kmer_span - 1 assert(seq_names.size() == num_seq); startNewSequence = true; } else { // sequence line if (startNewSequence) { if (line.length() < settings.kmer_span) continue; // ignore sequences shorter than K start_sequence(line, tailingKmer, sequencePosition, settings); startNewSequence = false; } else continue_sequence(line, tailingKmer, sequencePosition, settings); } } infile.close(); ids = added_ids; return added_ids.size(); } /* Start adding all k-mers in a sequence string s to the database. A new ID is created for this sequence. Return: sequence ID */ GenomeIdType KixBuild::start_sequence(const std::string &s, std::string& tailingKmer, PositionType& sequencePosition, AlignmentSettings & settings) { assert(seq_names.size() == num_seq); assert(s.length() >= settings.kmer_span); // add sequence kmers to index sequencePosition = 0; // use 1-based positions (to allow for negative positions) std::string::const_iterator it_s = s.begin(); std::string::const_iterator last_invalid; HashIntoType fw; // forward k-mer seq_lengths.back()=settings.kmer_span-1; for (; it_s < s.end()-settings.kmer_span+1; ++it_s) { ++sequencePosition; // use 1-based positions (to allow for negative positions) seq_lengths.back()+=1; last_invalid = hash_fw(it_s, s.end(), fw, settings); // add k-mer to database if (last_invalid < it_s) add_kmer(fw,num_seq-1,sequencePosition); else { unsigned jumplength = std::min(last_invalid - it_s, s.end() - settings.kmer_span - it_s); sequencePosition += jumplength; seq_lengths.back() += jumplength; it_s = last_invalid; } } tailingKmer = s.substr(s.length()-settings.kmer_span); return num_seq; } /* Continue adding all k-mers in a sequence string s to the database. Return: sequence ID */ GenomeIdType KixBuild::continue_sequence(const std::string &s, std::string& tailingKmer, PositionType& sequencePosition, AlignmentSettings & settings) { assert(seq_names.size() == num_seq); std::string concatString = tailingKmer + s; // add sequence kmers to index std::string::const_iterator it_s = concatString.begin() + 1; std::string::const_iterator last_invalid; HashIntoType fw; // forward k-mer for (; it_s < concatString.end()-settings.kmer_span+1; ++it_s) { ++sequencePosition; // use 1-based positions (to allow for negative positions) seq_lengths.back()+=1; last_invalid = hash_fw(it_s, concatString.end(), fw, settings); // add k-mer to database if (last_invalid < it_s) add_kmer(fw,num_seq-1,sequencePosition); else { unsigned jumplength = std::min(last_invalid - it_s, concatString.end()- settings.kmer_span - it_s); sequencePosition += jumplength; seq_lengths.back() += jumplength; it_s = last_invalid; } } tailingKmer = concatString.substr(concatString.length()-settings.kmer_span); return num_seq; } int KixBuild::add_kmer(HashIntoType kmer, GenomeIdType id, PositionType pos) { assert(kmer < db.size()); assert(id < num_seq); GenomePosType gp; gp.gid = id; gp.pos = pos; db[kmer].push_back(gp); return 1; } /* Trim the k-mer index: removes all k-mers with more than max_count occurrences in the reference genomes. Trimmed k-mers are marked by the GenomeIdType TRIMMED (from definitions.h). */ uint64_t KixBuild::trim(uint64_t max_count) { uint64_t trimmed = 0; GenomePosType gp_trimmed (TRIMMED,0); for (auto it = db.begin(); it != db.end(); ++it) if ((*it).size() > max_count) { trimmed += (*it).size(); (*it).clear(); (*it).push_back(gp_trimmed); } return trimmed; } std::vector KixBuild::serialize() { // first of all, sort the database entries by position for (auto it = db.begin(); it != db.end(); ++it) std::sort(it->begin(), it->end(), gp_compare); // calculate total size unsigned long int total_size = 0; // K_HiLive itself total_size += 1; // total number of sequences in database total_size += sizeof(GenomeIdType); // sequence names for (uint32_t i = 0; i < seq_names.size(); i++) { uint16_t nm_length = seq_names.size(); total_size += sizeof(uint16_t); total_size += nm_length; } // reference sequence lengths for (uint32_t i = 0; i < seq_lengths.size(); i++) { total_size += sizeof(uint64_t); } // database entries for (auto it = db.begin(); it != db.end(); ++it) { total_size += sizeof(uint32_t); // number of positions uint32_t num_positions = (*it).size(); total_size += num_positions*(sizeof(GenomeIdType) + sizeof(PositionType)); } // create the vector to store the data std::vector data (total_size); char* d = data.data(); // write K uint8_t kk = K_HiLive; memcpy(d,&kk,1); d++; // total number of sequences in database memcpy(d,&num_seq,sizeof(GenomeIdType)); d += sizeof(GenomeIdType); // sequence names for (uint32_t i = 0; i < seq_names.size(); i++) { uint16_t nm_length = seq_names[i].size(); memcpy(d,&nm_length,sizeof(uint16_t)); d += sizeof(uint16_t); memcpy(d,seq_names[i].c_str(),nm_length); d += nm_length; } for (uint32_t i = 0; i < seq_lengths.size(); i++) { uint64_t seq_len = seq_lengths[i]; memcpy(d,&seq_len,sizeof(uint64_t)); d += sizeof(uint64_t); } // database entries for (auto it = db.begin(); it != db.end(); ++it) { // number of positions uint32_t num_positions = (*it).size(); //db is an array of GenomePosType structs memcpy(d,&num_positions,sizeof(uint32_t)); d += sizeof(uint32_t); // genome ID and position for(GenomePosListIt entry=(*it).begin(); entry!=(*it).end(); ++entry) { GenomeIdType gid = (*entry).gid; PositionType pos = (*entry).pos; memcpy(d,&gid,sizeof(GenomeIdType)); d += sizeof(GenomeIdType); memcpy(d,&pos,sizeof(PositionType)); d += sizeof(PositionType); } } return data; } uint64_t KixBuild::serialize_file(std::string f) { std::string fname = f; // serialize data std::vector sdata = serialize(); // open binary file FILE* ofile; ofile = fopen(fname.c_str(), "wb"); if (!ofile) { std::cerr << "Error serializing object to file " << fname << ": Could not open file for writing." << std::endl; return 0; } // write all data uint64_t written = fwrite(sdata.data(), 1, sdata.size(), ofile); // close file fclose(ofile); if (written != sdata.size()){ std::cerr << "Error serializing object to file " << fname << ": Total size: " << sdata.size() << " bytes. Written: " << written << " bytes." << std::endl; } return written; } uint64_t KixBuild::deserialize(char* d) { // the total number of bytes read uint64_t bytes = 0; // read and check K uint8_t kk; memcpy(&kk,d+bytes,1); bytes++; assert(K_HiLive == kk); // read total number of sequences in database memcpy(&num_seq,d+bytes,sizeof(GenomeIdType)); bytes += sizeof(GenomeIdType); // sequence names seq_names.clear(); seq_lengths.clear(); for (uint32_t i = 0; i < num_seq; i++) { uint16_t nm_length; memcpy(&nm_length,d+bytes,sizeof(uint16_t)); bytes += sizeof(uint16_t); char * tmp = new char[nm_length+1]; memcpy(tmp,d+bytes,nm_length); tmp[nm_length] = 0; // make the string null-terminated seq_names.push_back(tmp); delete tmp; bytes += nm_length; } for (uint32_t i = 0; i < num_seq; i++) { uint64_t seq_len; memcpy(&seq_len,d+bytes,sizeof(uint64_t)); bytes += sizeof(uint64_t); seq_lengths.push_back(seq_len); } // database entries for (auto it = db.begin(); it != db.end(); ++it) { // number of positions uint32_t num_positions; memcpy(&num_positions,d+bytes,sizeof(uint32_t)); bytes += sizeof(uint32_t); (*it).clear(); (*it).reserve(num_positions); // genome ID and position for( uint32_t i = 0; i < num_positions; i++) { GenomeIdType gid; PositionType pos; GenomePosType gp; memcpy(&gid,d+bytes,sizeof(GenomeIdType)); bytes += sizeof(GenomeIdType); memcpy(&pos,d+bytes,sizeof(PositionType)); bytes += sizeof(PositionType); gp.gid = gid; gp.pos = pos; (*it).push_back(gp); } } return bytes; } uint64_t KixBuild::deserialize_file(std::string f) { std::string fname = f; // obtain file size unsigned long int size = get_filesize(fname); // open binary file FILE* ifile; ifile = fopen(fname.c_str(), "rb"); if (!ifile) { std::cerr << "Error reading from file " << fname << ": Could not open file." << std::endl; return 0; } // allocate memory std::vector sdata (size); // read all data unsigned long int read = fread(sdata.data(), 1, size, ifile); // close file fclose (ifile); if (read != size){ std::cerr << "Error reading from file " << fname << ": File size: " << size << " bytes. Read: " << read << " bytes." << std::endl; return 0; } // deserialize data deserialize(sdata.data()); return read; } uint64_t KixRun::deserialize(char* d) { // the total number of bytes read uint64_t bytes = 0; // read and check K uint8_t kk; memcpy(&kk,d+bytes,1); bytes++; assert(kk == K_HiLive); // read total number of sequences in database memcpy(&num_seq,d+bytes,sizeof(GenomeIdType)); bytes += sizeof(GenomeIdType); // sequence names seq_names.clear(); seq_lengths.clear(); for (uint32_t i = 0; i < num_seq; i++) { uint16_t nm_length; memcpy(&nm_length,d+bytes,sizeof(uint16_t)); bytes += sizeof(uint16_t); char * tmp = new char[nm_length+1]; memcpy(tmp,d+bytes,nm_length); tmp[nm_length] = 0; // make the string null-terminated seq_names.push_back(tmp); delete tmp; bytes += nm_length; } // sequence lengths for (uint32_t i = 0; i < num_seq; i++) { uint64_t seq_len; memcpy(&seq_len,d+bytes,sizeof(uint64_t)); bytes += sizeof(uint64_t); seq_lengths.push_back(seq_len); } // database entries for (auto it = db.begin(); it != db.end(); ++it) { // number of positions uint32_t* num_positions = (uint32_t*)(d+bytes); // total size of data block for this k-mer uint32_t total_size = sizeof(uint32_t) + (*num_positions)*GenomePos_size; // allocate the memory (*it) = d+bytes; // increase pointer bytes += total_size; } return bytes; } uint64_t KixRun::deserialize_file(std::string f) { std::string fname = f; sdata = read_binary_file(f); deserialize(sdata.data()); return sdata.size(); } /* Retrieve all occurrences (fwd & rc) of kmer in the reference from the index */ GenomePosListType KixRun::retrieve_positions(std::string kmerSpan, AlignmentSettings & settings) { // get the reverse complement of the kmer HashIntoType fwHashValue; HashIntoType rcHashValue; hash(kmerSpan.c_str(), fwHashValue, rcHashValue, settings); // obtain the list of positions for each k-mer char* fwd_begin = db[fwHashValue]; uint32_t fwd_len; memcpy(&fwd_len,fwd_begin,sizeof(uint32_t)); char* rev_begin = db[rcHashValue]; uint32_t rev_len; memcpy(&rev_len,rev_begin,sizeof(uint32_t)); // the position list: all positions in all genomes, where the current k-mer was found GenomePosListType pos; pos.reserve(fwd_len+rev_len); // indicate reverse complement hits by negative position and append in reverse order for(uint64_t i = 0; i < rev_len; i++) { GenomePosType rev; memcpy(&rev.gid,rev_begin+sizeof(uint32_t)+(rev_len-1-i)*GenomePos_size, sizeof(GenomeIdType)); memcpy(&rev.pos,rev_begin+sizeof(uint32_t)+(rev_len-1-i)*GenomePos_size+sizeof(GenomeIdType), sizeof(PositionType)); rev.pos = - rev.pos; if (rev.gid == TRIMMED) { pos.clear(); pos.push_back(GenomePosType(TRIMMED,0)); return pos; } pos.push_back(rev); } // then add the forward hits with positive position in normal order --> position list is sorted if index was sorted! for(uint64_t i = 0; i < fwd_len; i++) { GenomePosType fwd; memcpy(&fwd.gid,fwd_begin+sizeof(uint32_t)+i*GenomePos_size, sizeof(GenomeIdType)); memcpy(&fwd.pos,fwd_begin+sizeof(uint32_t)+i*GenomePos_size+sizeof(GenomeIdType), sizeof(PositionType)); if (fwd.gid == TRIMMED) { pos.clear(); pos.push_back(GenomePosType(TRIMMED,0)); return pos; } pos.push_back(fwd); } return pos; } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/kindex.h000066400000000000000000000056531301327033400225470ustar00rootroot00000000000000#ifndef KINDEX_H #define KINDEX_H #include "headers.h" #include "definitions.h" #include "tools.h" //-------------------------------------------------------------------// //------ The k-mer index builder: KixBuild -------------------------// //-------------------------------------------------------------------// class KixBuild { // add a single k-mer to the database // Note: the index uses 1-based positions (to allow for negative positions) int add_kmer(HashIntoType kmer, GenomeIdType id, PositionType pos); public: // add k-mers of all sequences in FASTA file int add_fasta(const std::string &fname, GenomeIdListType &ids, AlignmentSettings & settings, bool convert_spaces, bool trim_ids); int add_fasta(const std::string &fname, AlignmentSettings & settings, bool convert_spaces, bool trim_ids); // add all k-mers in a string sequence to the database GenomeIdType start_sequence(const std::string &s, std::string& tailingKmer, PositionType& sequencePosition, AlignmentSettings & settings); GenomeIdType continue_sequence(const std::string &s, std::string& tailingKmer, PositionType& sequencePosition, AlignmentSettings & settings); // trim the database: remove kmers with more than max_count occurrences uint64_t trim(uint64_t max_count); // serialize the KixBuild std::vector serialize(); // serialize and store the KixBuild to a file uint64_t serialize_file(std::string f); // deserialize KixBuild uint64_t deserialize(char* d); // load and deserialize KixBuild from file uint64_t deserialize_file(std::string f); GenomeIdType num_seq; // total number of sequences in the database KmerIndexType db; // the database structure itself StringListType seq_names; // names of the sequences in the database std::vector seq_lengths; // lengths of the sequences in the database }; // END class KixBuild //-------------------------------------------------------------------// //------ The k-mer runtime index: KixRun ---------------------------// //-------------------------------------------------------------------// typedef std::array KixRunDB; class KixRun { public: // pointer to the matching positions for a k-mer char* kmer(HashIntoType kmer); // retrieve all fwd and rc occurrences of kmer in the index GenomePosListType retrieve_positions(std::string kmerSpan, AlignmentSettings & settings); // deserialize Kix uint64_t deserialize(char* d); // load and deserialize Kix from file uint64_t deserialize_file(std::string f); // Database content GenomeIdType num_seq; // total number of sequences in the database StringListType seq_names; // names of the sequences in the database std::vector seq_lengths; // lengths of the sequences in the database KixRunDB db; // the lightweight database structure itself, pointing to sdata std::vector sdata; // actual chunk of data }; // END class KixRun #endif /* KINDEX_H */ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/parallel.cpp000066400000000000000000000170031301327033400234040ustar00rootroot00000000000000#include "parallel.h" std::ostream& operator<<(std::ostream& os, const Task& t) { os << "Lane " << t.lane << " Tile " << t.tile << " Cycle " << t.cycle; return os; } // Add element to the task list void TaskQueue::push(Task t) { std::lock_guard lk(m); tasks.push(t); } // Get element from the task list. If TaskList is empty, return NO_TASK. Task TaskQueue::pop() { std::lock_guard lk(m); if (!tasks.empty()) { Task t = tasks.front(); tasks.pop(); return t; } else { return NO_TASK; } } // return the size of the queue uint64_t TaskQueue::size() { std::lock_guard lk(m); return tasks.size(); } // create a vector with all lane numbers std::vector all_lanes() { std::vector ln; for (uint16_t l=0; l < 8; l++) ln.push_back(l+1); return ln; } // create a vector with one lane number std::vector one_lane(uint16_t l) { return std::vector (1,l); } // create a vector with all tile numbers std::vector all_tiles() { std::vector tl; for (uint16_t l = 0; l < 2; l++) { for (uint16_t s = 0; s < 3; s++) { for (uint16_t t = 0; t < 16; t++) { // construct tile number tl.push_back( (l+1)*1000 + (s+1)*100 + (t+1) ); } } } return tl; } // create a vector with one tile number std::vector one_tile(uint16_t t) { return std::vector (1,t); } // initialize agenda with root directory and read length only (all lanes, all tiles) Agenda::Agenda (std::string rt, uint16_t rl) { // add lanes 1-8 to the list std::vector ln = all_lanes(); // call the tiles constructor Agenda(rt, rl, ln); } // initialize agenda with root directory, read length, and lanes (all tiles) Agenda::Agenda (std::string rt, uint16_t rl, std::vector ln) { // add all tiles to the list std::vector tl = all_tiles(); // call the full constructor Agenda (rt, rl, ln, tl); } // initialize agenda with root directory, read length, lanes, and tiles Agenda::Agenda (std::string rt, uint16_t rl, std::vector ln, std::vector tl) { root = rt; rlen = rl; lanes = ln; tiles = tl; // set up the agenda items.clear(); for (uint16_t ln_id = 0; ln_id < lanes.size(); ln_id++) { std::vector > lane_status; for (uint16_t tl_id = 0; tl_id < tiles.size(); tl_id++) { std::vector tile_status (rlen, WAITING); lane_status.push_back(tile_status); } items.push_back(lane_status); } } // check for BCL files and update item status void Agenda::update_status () { // iterate over lanes for (uint16_t ln_id = 0; ln_id < items.size(); ++ln_id) { // iterate over all tiles for (uint16_t tl_id = 0; tl_id < items[ln_id].size(); ++tl_id) { // get the first cycle that is not in the FINISHED status uint16_t first_unfinished = 0; while ( (first_unfinished < items[ln_id][tl_id].size()) && (items[ln_id][tl_id][first_unfinished] == FINISHED)) { first_unfinished++; } // if there is one, check if there is a BCL file available if ((first_unfinished != items[ln_id][tl_id].size()) && (items[ln_id][tl_id][first_unfinished] == WAITING)) { std::string this_fname = bcl_name(root, lanes[ln_id], tiles[tl_id], first_unfinished+1); // only change the status if the file exists if ( file_exists(this_fname) ) { // TODO: probably find a way to check if the machine currently writes to that file items[ln_id][tl_id][first_unfinished] = BCL_AVAILABLE; } } } } } // generate a new task from the agenda Task Agenda::get_task(){ // iterate over lanes for (uint16_t ln_id = 0; ln_id < items.size(); ++ln_id) { // iterate over all tiles for (uint16_t tl_id = 0; tl_id < items[ln_id].size(); ++tl_id) { // check if there is a cycle with an unprocessed BCL file uint16_t unprocessed = 0; while ( (unprocessed < items[ln_id][tl_id].size()) && (items[ln_id][tl_id][unprocessed] != BCL_AVAILABLE)) { unprocessed++; } // generate a new task if there is an unprocessed BCL file if ( unprocessed != items[ln_id][tl_id].size() ) { Task t (lanes[ln_id], tiles[tl_id], unprocessed+1, rlen, root); return t; } } } // return indicator that no new task could be created return NO_TASK; } // set a status void Agenda::set_status(Task t, ItemStatus status) { // get the lane index uint64_t diff = std::find(lanes.begin(), lanes.end(), t.lane) - lanes.begin(); if ( diff >= lanes.size() ) { throw std::out_of_range("Lane ID out of range."); } uint16_t ln_id = diff; // get the tile index diff = std::find(tiles.begin(), tiles.end(), t.tile) - tiles.begin(); if ( diff >= tiles.size() ) { throw std::out_of_range("Tile ID out of range."); } uint16_t tl_id = diff; // get the cycle index if ( (t.cycle > rlen) || (t.cycle == 0) ) { throw std::out_of_range("Cycle out of range."); } uint16_t cl_id = t.cycle -1; items[ln_id][tl_id][cl_id] = status; } // get the status of a task ItemStatus Agenda::get_status(Task t) { // get the lane index uint64_t diff = std::find(lanes.begin(), lanes.end(), t.lane) - lanes.begin(); if ( diff >= lanes.size() ) { throw std::out_of_range("Lane ID out of range."); } uint16_t ln_id = diff; // get the tile index diff = std::find(tiles.begin(), tiles.end(), t.tile) - tiles.begin(); if ( diff >= tiles.size() ) { throw std::out_of_range("Tile ID out of range."); } uint16_t tl_id = diff; // get the cycle index if ( (t.cycle > rlen) || (t.cycle == 0) ) { throw std::out_of_range("Cycle out of range."); } uint16_t cl_id = t.cycle -1; return items[ln_id][tl_id][cl_id]; } // check if all items of the agenda were processed, if possible bool Agenda::finished() { // check for each tile if either all cycles are finished OR there is a failed status item for (uint16_t ln_id = 0; ln_id < items.size(); ++ln_id) { for (uint16_t tl_id = 0; tl_id < items[ln_id].size(); ++tl_id) { for (uint16_t cl_id = 0; cl_id < items[ln_id][tl_id].size(); ++cl_id) { ItemStatus s = items[ln_id][tl_id][cl_id]; if ( s == FAILED ) { // the rest of the tile is "allowed" to be unprocessed --> skip continue; } else if (s != FINISHED) { // otherwise any other status means that the agenda is not finished return false; } } } } return true; } // the total number of tasks on the agenda uint32_t Agenda::task_count() { return lanes.size() * tiles.size() * rlen; } // the total number of finished tasks on the agenda uint32_t Agenda::tasks_finished() { uint32_t num_finished = 0; // iterate over all items and count the finished tasks for (uint16_t ln_id = 0; ln_id < items.size(); ++ln_id) { for (uint16_t tl_id = 0; tl_id < items[ln_id].size(); ++tl_id) { for (uint16_t cl_id = 0; cl_id < items[ln_id][tl_id].size(); ++cl_id) { if (items[ln_id][tl_id][cl_id] == FINISHED) { num_finished++; } } } } return num_finished; } // generate a complete TaskQueue with tasks to generate SAM files // SAM files can only be generated for tiles where all cycles are completed std::vector Agenda::get_SAM_tasks() { std::vector tv; // find tiles that are completely mapped for (uint16_t ln_id = 0; ln_id < items.size(); ++ln_id) { for (uint16_t tl_id = 0; tl_id < items[ln_id].size(); ++tl_id) { if ( items[ln_id][tl_id][rlen-1] == FINISHED ) { tv.push_back(Task(lanes[ln_id],tiles[tl_id],rlen,rlen,root)); } } } return tv; } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/parallel.h000066400000000000000000000070361301327033400230560ustar00rootroot00000000000000#ifndef PARALLEL_H #define PARALLEL_H #include "headers.h" #include "definitions.h" #include "tools.h" #include "kindex.h" //------ Threading tools --------------------------------------------// // Task data structure. Contains all information for a thread to // process a BCL file. struct Task { // dataset information uint16_t lane; uint16_t tile; uint16_t cycle; uint16_t rlen; std::string root; // constructor initializes all member variables Task(uint16_t ln, uint16_t tl, uint16_t cl, uint16_t rl, std::string rt): lane(ln), tile(tl), cycle(cl), rlen(rl), root(rt) {}; // overload << operator for printing friend std::ostream& operator<<(std::ostream& os, const Task& t); }; inline bool operator==(const Task& l, const Task& r){ return (r.lane==l.lane)&&(r.tile==l.tile)&&(r.cycle==l.cycle)&&(r.rlen==l.rlen)&&(r.root==l.root); } inline bool operator!=(const Task& l, const Task& r){ return !(l==r); } const Task NO_TASK (255,0,0,0,""); // Task queue data structure. Manages a list of task objects in a thread safe way. class TaskQueue { // the internal queue std::queue tasks; // mutex to ensure that only one process can access the queue at once std::mutex m; public: // Add element to the task list void push(Task t); // Get element from the task list Task pop(); // return the size of the queue uint64_t size(); }; // Agenda item status typedef uint8_t ItemStatus; const ItemStatus WAITING = 0; const ItemStatus BCL_AVAILABLE = 1; const ItemStatus RUNNING = 2; const ItemStatus FINISHED = 3; const ItemStatus RETRY = 4; const ItemStatus FAILED = 5; const ItemStatus ERROR = std::numeric_limits::max(); // Agenda: monitors the sequencing process and manages the alignment // - Monitors BCL files // - generates new tasks // - receive finished/fail signals class Agenda { // list of items on the agenda. items[lane][tile][cycle] std::vector< std::vector< std::vector > > items; // dataset information std::string root; uint16_t rlen; std::vector lanes; std::vector tiles; public: // initialize agenda with root directory and read length only (all lanes, all tiles) Agenda (std::string rt, uint16_t rl); // initialize agenda with root directory, read length, and lanes (all tiles) Agenda (std::string rt, uint16_t rl, std::vector ln); // initialize agenda with root directory, read length, lanes, and tiles Agenda (std::string rt, uint16_t rl, std::vector ln, std::vector tl); // check for BCL files and update item status void update_status(); // generate a new task from the agenda Task get_task(); // set the status of a task void set_status(Task t, ItemStatus status); // get the status of a task ItemStatus get_status(Task t); // check if all items of the agenda were processed, if possible bool finished(); // the total number of tasks on the agenda uint32_t task_count(); // the total number of finished tasks on the agenda uint32_t tasks_finished(); // generate a complete TaskQueue with tasks to generate SAM files // SAM files can only be generated for tiles where all cycles are completed std::vector get_SAM_tasks(); }; // create a vector with all lane numbers std::vector all_lanes(); // create a vector with one lane number std::vector one_lane(uint16_t l); // create a vector with all tile numbers std::vector all_tiles(); // create a vector with one tile number std::vector one_tile(uint16_t t); #endif /* PARALLEL_H */ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/tools.cpp000066400000000000000000000150131301327033400227470ustar00rootroot00000000000000#include "tools.h" // compares two genome positions by position (not by genome id) bool gp_compare (GenomePosType i,GenomePosType j) { return (i.pos < j.pos); } // reads a binary file from hdd and stores its raw content in a char vector std::vector read_binary_file(const std::string &fname) { // get file size uint64_t size = get_filesize(fname); // open binary file FILE* f; f = fopen(fname.c_str(), "rb"); if (!f) { std::cerr << "Error reading binary file " << fname << ": Could not open file." << std::endl; return std::vector(); } // allocate memory std::vector data (size); // read all data at once uint64_t read = fread(data.data(), 1, size, f); if (read != size){ std::cerr << "Error reading binary file " << fname << ": File size: " << size << " bytes. Read: " << read << " bytes." << std::endl; return std::vector(); } fclose(f); return data; } // checks if a directory with the given path exists bool is_directory(const std::string &path) { if ( boost::filesystem::exists(path) ) { if ( boost::filesystem::is_directory(path) ) { return true; } else { return false; } } else { return false; } } // checks if a file exists bool file_exists(const std::string &fname) { return boost::filesystem::exists(fname); /*if (FILE *f = fopen(fname.c_str(), "r")) { fclose(f); return true; } else { return false; }*/ } // writes a char vector into a binary file uint64_t write_binary_file(const std::string &fname, const std::vector & data) { // open binary file FILE* ofile; ofile = fopen(fname.c_str(), "wb"); if (!ofile) { std::cerr << "Error serializing object to file " << fname << ": Could not open file for writing." << std::endl; return 0; } // write all data uint64_t written = fwrite(data.data(), 1, data.size(), ofile); // close file fclose(ofile); if (written != data.size()){ std::cerr << "Error serializing object to file " << fname << ": Total size: " << data.size() << " bytes. Written: " << written << " bytes." << std::endl; } return written; } // extract the number of reads from a BCL file uint32_t num_reads_from_bcl(std::string bcl) { // open BCL file of first cycle FILE* ifile; ifile = fopen(bcl.c_str(), "rb"); if (!ifile) { std::cerr << "Error reading BCL file " << bcl << ": Could not open file." << std::endl; return 0; } // extract the number of reads uint32_t num_reads; assert(fread(&num_reads, 1, sizeof(uint32_t), ifile)); // close file fclose (ifile); return num_reads; } /* get the size (in bytes) of a file */ std::ifstream::pos_type get_filesize(const std::string &fname) { std::ifstream in(fname, std::ios::binary | std::ios::ate); return in.tellg(); } /* calculates the first forward and reverse complement k-mer in the string and returns the canonical representation. */ HashIntoType hash(const char * kmer, HashIntoType& _h, HashIntoType& _r, AlignmentSettings & settings) { assert(strlen(kmer) >= settings.kmer_span); HashIntoType h = 0, r = 0; h |= twobit_repr(kmer[0]); r |= twobit_comp(kmer[settings.kmer_span-1]); for (unsigned int i = 1, j = settings.kmer_span-2; i < settings.kmer_span; i++, j--) { // if i not gap position if (std::find(settings.kmer_gaps.begin(), settings.kmer_gaps.end(), i+1) == settings.kmer_gaps.end()) { h = h << 2; h |= twobit_repr(kmer[i]); r = r << 2; r |= twobit_comp(kmer[j]); } } _h = h; _r = r; return (h)<(r)?h:r; } /* calculates the first forward k-mer in the string */ std::string::const_iterator hash_fw(std::string::const_iterator it, std::string::const_iterator end, HashIntoType& _h, AlignmentSettings & settings) { assert(it+settings.kmer_span-1 < end); HashIntoType h = 0; std::string::const_iterator last_invalid = it-1; h |= twobit_repr(*it); std::string::const_iterator kmerEnd = it+settings.kmer_span; ++it; int positionInKmer = 2; for (; it != kmerEnd; ++it, ++positionInKmer) { if (std::find(settings.kmer_gaps.begin(), settings.kmer_gaps.end(), positionInKmer) != settings.kmer_gaps.end()) continue; h = h << 2; h |= twobit_repr(*it); if ( seq_chars.find(*it) == std::string::npos ) { last_invalid = it+settings.kmer_span-1; } } _h = h; return last_invalid; } /* returns the sequence of a k-mer */ std::string unhash(HashIntoType myHash, unsigned hashLen) { std::string kmer = ""; unsigned mask = 3; for (unsigned i = 1; i> 2; } std::reverse(kmer.begin(), kmer.end()); return kmer; } // file name construction functions // construct BCL file name from: root, lane, tile, cycle std::string bcl_name(std::string rt, uint16_t ln, uint16_t tl, uint16_t cl) { std::ostringstream path_stream; path_stream << rt << "/L00" << ln << "/C" << cl << ".1/s_"<< ln <<"_" << tl << ".bcl"; return path_stream.str(); } // construct alignment file name from: root, lane, tile, cycle std::string alignment_name(std::string rt, uint16_t ln, uint16_t tl, uint16_t cl) { std::ostringstream path_stream; path_stream << rt << "/L00" << ln << "/s_"<< ln << "_" << tl << "." << cl << ".align"; return path_stream.str(); } // construct tile-wise SAM file name from: root, lane, tile std::string sam_tile_name(std::string rt, uint16_t ln, uint16_t tl, bool write_bam) { std::ostringstream path_stream; if (write_bam) path_stream << rt << "/L00" << ln << "/s_"<< ln << "_" << tl << ".bam"; else path_stream << rt << "/L00" << ln << "/s_"<< ln << "_" << tl << ".sam"; return path_stream.str(); } // construct lane-wise SAM file name from: root, lane std::string sam_lane_name(std::string rt, uint16_t ln, bool write_bam) { std::ostringstream path_stream; if (write_bam) path_stream << rt << "/L00" << ln << "/s_"<< ln << ".bam"; else path_stream << rt << "/L00" << ln << "/s_"<< ln << ".sam"; return path_stream.str(); } // construct filter file name from: root, lane, tile std::string filter_name(std::string rt, uint16_t ln, uint16_t tl) { std::ostringstream path_stream; path_stream << rt << "/L00" << ln << "/s_"<< ln << "_" << tl << ".filter"; return path_stream.str(); } // construct position file name from: root, lane, tile std::string position_name(std::string rt, uint16_t ln, uint16_t tl) { std::ostringstream path_stream; path_stream << rt << "../L00" << ln << "/s_"<< ln << "_" << tl << ".clocs"; return path_stream.str(); } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/lib/tools.h000066400000000000000000000034621301327033400224210ustar00rootroot00000000000000#ifndef TOOLS_H #define TOOLS_H #include "headers.h" #include "definitions.h" #include "kindex.h" // compare function to sort GenomePosType objects by position bool gp_compare (GenomePosType i,GenomePosType j); // calculates the total size of a file in bytes std::ifstream::pos_type get_filesize(const std::string &fname); // checks if a directory with the given name exists bool is_directory(const std::string &path); // checks if a file exists bool file_exists(const std::string &fname); // reads a binary file from hdd and stores its raw content in a char vector std::vector read_binary_file(const std::string &fname); // extract the number of reads from a BCL file uint32_t num_reads_from_bcl(std::string bcl); // writes a char vector into a binary file uint64_t write_binary_file(const std::string &fname, const std::vector & data); //------ Hashing helper functions ---------------------------------// HashIntoType hash(const char * kmer, HashIntoType& _h, HashIntoType& _r, AlignmentSettings & settings); std::string::const_iterator hash_fw(std::string::const_iterator it, std::string::const_iterator end, HashIntoType& _h, AlignmentSettings & settings); //HashIntoType rc(HashIntoType fw); /* returns the sequence of a k-mer */ std::string unhash(HashIntoType myHash, unsigned hashLen=K_HiLive); // file name construction functions std::string bcl_name(std::string rt, uint16_t ln, uint16_t tl, uint16_t cl); std::string alignment_name(std::string rt, uint16_t ln, uint16_t tl, uint16_t cl); std::string filter_name(std::string rt, uint16_t ln, uint16_t tl); std::string position_name(std::string rt, uint16_t ln, uint16_t tl); std::string sam_tile_name(std::string rt, uint16_t ln, uint16_t tl, bool write_bam); std::string sam_lane_name(std::string rt, uint16_t ln, bool write_bam); #endif /* TOOLS_H */ HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/tools/000077500000000000000000000000001301327033400214755ustar00rootroot00000000000000HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/tools/build_index.cpp000066400000000000000000000131161301327033400244710ustar00rootroot00000000000000#include #include "../lib/headers.h" #include "../lib/definitions.h" #include "../lib/kindex.h" namespace po = boost::program_options; std::string license = "Copyright (c) 2015-2016, Martin S. Lindner and the HiLive contributors. See CONTRIBUTORS for more info.\n" "All rights reserved.\n" "\n" "Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n" "\n" "1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n" "\n" "2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n" "\n" "3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.\n" "\n" "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."; int main(int argc, char* argv[]) { unsigned trim; bool do_not_convert_spaces; bool trim_ids; // setting up the command line interface po::options_description general("General"); general.add_options() ("help,h", "Print this help message and exit") ("license", "Print licensing information and exit"); po::options_description parameters("Parameters"); parameters.add_options() ("INPUT", po::value()->required(), "Input reference genome (fasta file)"); po::options_description options("Options"); options.add_options() ("outfile,o", po::value(), "Set output file name [Default: INPUT.kix]") ("trim,t", po::value(&trim)->default_value(0), "Ignore k-mers with more than t occurrences. [Default: no limit]") ("do-not-convert-spaces", po::bool_switch(&do_not_convert_spaces)->default_value(false), "Do not convert all spaces in reference ids to underscores [Default: converting is on]") ("trim-after-space", po::bool_switch(&trim_ids)->default_value(false), "Trim all reference ids after first space [Default: false]"); po::options_description cmdline_options; cmdline_options.add(general).add(parameters).add(options); po::options_description visible_options; visible_options.add(general).add(options); std::stringstream help_message; help_message << "HiLive index builder v"<< HiLive_VERSION_MAJOR << "." << HiLive_VERSION_MINOR << std::endl; help_message << "Index creation tool for HiLive - Realtime Alignment of Illumina Reads" << std::endl; help_message << "Copyright (c) 2015, Martin S. Lindner" << std::endl; help_message << "HiLive is open-source software. Check with --license for details." << std::endl << std::endl; help_message << "Fixed k-mer size: " << K_HiLive << std::endl << std::endl; help_message << "Usage: " << std::string(argv[0]) << " [options] INPUT" << std::endl; help_message << " INPUT Reference genomes in (multi-) FASTA format" << std::endl; help_message << visible_options << std::endl; po::positional_options_description p; p.add("INPUT", 1); po::variables_map vm; try { // parse arguments po::store(po::command_line_parser(argc, argv). options(cmdline_options).positional(p).run(), vm); // first check if -h or --help was called if (vm.count("help")) { std::cout << help_message.str(); return 1; } // first check if --license was called if (vm.count("license")) { std::cout << license << std::endl; return 1; } // then check arguments po::notify(vm); } catch ( boost::program_options::required_option& e ) { std::cerr << "Missing Parameter: " << e.what() << std::endl << std::endl; std::cout << help_message.str(); return -1; } catch( boost::program_options::error& e) { std::cerr << "Error while parsing command line options: " << e.what() << std::endl << std::endl; std::cout << help_message.str(); return -1; } std::string fasta_name = vm["INPUT"].as(); std::string index_name; if (vm.count("outfile")) { // use the output file name if provided as argument index_name = vm["outfile"].as(); } else { // construct it from the input file name otherwise index_name = fasta_name + std::string(".kix"); } std::cout << "Creating index with K_HiLive=" << K_HiLive << " from file " << fasta_name << std::endl; KixBuild* index = new KixBuild(); AlignmentSettings settings; // for hard coded gapped kmer structure index->add_fasta(fasta_name, settings, !do_not_convert_spaces, trim_ids); if (trim > 0) { uint64_t trimmed = index->trim(trim); std::cout << "Removed " << trimmed << " k-mer positions from the database." << std::endl; } std::cout << "Writing index to file " << index_name << std::endl; index->serialize_file(index_name); delete index; } HiLive-v0.3-ce80f3a2af1d5d0d327c5b40562bd077477f80b0/tools/hilive.cpp000066400000000000000000000176731301327033400234770ustar00rootroot00000000000000#include "../lib/headers.h" #include "../lib/definitions.h" #include "../lib/kindex.h" #include "../lib/alnstream.h" #include "../lib/parallel.h" #include "../lib/argument_parser.h" std::string license = "Copyright (c) 2015-2016, Martin S. Lindner and the HiLive contributors. See CONTRIBUTORS for more info.\n" "All rights reserved.\n" "\n" "Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n" "\n" "1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n" "\n" "2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n" "\n" "3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.\n" "\n" "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."; // the worker function for the threads void worker (TaskQueue & tasks, TaskQueue & finished, TaskQueue & failed, AlignmentSettings* settings, KixRun* idx, bool & surrender ) { // loop that keeps on running until the surrender flag is set while ( !surrender ) { // try to obtain a new task Task t = tasks.pop(); if ( t != NO_TASK ) { // Execute the task bool success = true; try { StreamedAlignment s (t.lane, t.tile, t.root, t.rlen); uint64_t num_seeds; num_seeds = s.extend_alignment(t.cycle,idx,settings); std::cout << "Task [" << t << "]: Found " << num_seeds << " seeds." << std::endl; } catch (const std::exception &e) { std::cerr << "Failed to finish task [" << t << "]: " << e.what() << std::endl; success = false; } if (success) { finished.push(t); } else { failed.push(t); } } else { // send this thread to sleep for a second std::this_thread::sleep_for (std::chrono::milliseconds(100)); } } } // create a special SAM worker, that writes out a SAM file for a tile void sam_worker (TaskQueue & tasks, AlignmentSettings* settings, KixRun* idx) { // loop that keeps on running until the surrender flag is set while ( true ) { // try to obtain a new task Task t = tasks.pop(); if ( t != NO_TASK ) { // Execute the task alignments_to_sam(t.lane,t.tile,t.root,t.rlen, idx,settings); } else { return; } } } int main(int argc, const char* argv[]) { time_t t_start = time(NULL); // parse the command line arguments, store results in settings AlignmentSettings settings; if (parseCommandLineArguments(settings, license, argc, argv)) // returns true if error occured return 1; // load the index std::cout << "Loading Index" << std::endl; KixRun* index = new KixRun(); index->deserialize_file(settings.index_fname); // Create the overall agenda Agenda agenda (settings.root, settings.rlen, settings.lanes, settings.tiles); // prepare the alignment std::cout << "Initializing Alignment files. Waiting for the first cycle to finish." << std::endl; bool first_cycle_available = false; // wait for the first cycle to be written. Attention - this loop will wait infinitely long if no first cycle is found while ( !first_cycle_available ) { // check for new BCL files and update the agenda status agenda.update_status(); // check if the first cycle is available for all tiles first_cycle_available = true; for ( auto ln : settings.lanes ) { for ( auto tl : settings.tiles ) { if ( agenda.get_status(Task(ln,tl,1,settings.rlen,"")) != BCL_AVAILABLE) { first_cycle_available = false; } } } // take a small break std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } std::cout << "First cycle complete. Starting alignment." << std::endl; // write empty alignment file for each tile for (uint16_t ln : settings.lanes) { for (uint16_t tl : settings.tiles) { StreamedAlignment s (ln, tl, settings.root, settings.rlen); s.create_directories(&settings); s.init_alignment(&settings); } } if (settings.temp_dir != "" && !is_directory(settings.temp_dir)){ std::cerr << "Error: Could not find temporary directory " << settings.temp_dir << std::endl; return -1; } // Set up the queues TaskQueue toDoQ; TaskQueue finishedQ; TaskQueue failedQ; // Create the threads std::cout << "Creating " << settings.num_threads << " threads." << std::endl; bool surrender = false; std::vector workers; for (int i = 0; i < settings.num_threads; i++) { workers.push_back(std::thread(worker, std::ref(toDoQ), std::ref(finishedQ), std::ref(failedQ), &settings, index, std::ref(surrender))); } // Process all tasks on the agenda while ( !agenda.finished() ) { // check for new BCL files and update the agenda status agenda.update_status(); // fill the ToDo queue with tasks from the agenda while(true) { Task t = agenda.get_task(); if (t == NO_TASK) break; toDoQ.push(t); agenda.set_status(t,RUNNING); } // take a look in the finished queue and process finished tasks while(true) { Task t = finishedQ.pop(); if (t == NO_TASK) break; agenda.set_status(t,FINISHED); } // take a look in the failed queue and process failed tasks while(true) { Task t = failedQ.pop(); if (t == NO_TASK) break; if (agenda.get_status(t) == RUNNING) { // give it one more chance agenda.set_status(t,RETRY); toDoQ.push(t); } else { agenda.set_status(t,FAILED); std::cout << "Task failed! " << t << std::endl; } } // take a small break std::this_thread::sleep_for (std::chrono::milliseconds(100)); } // Halt the threads surrender = true; for (auto& w : workers) { w.join(); } std::cout << "All threads joined." << std::endl; std::cout << "Writing SAM files." << std::endl; // Create individual SAM files for every tile TaskQueue sam_tasks; std::vector tv = agenda.get_SAM_tasks(); for ( auto t: tv ) { sam_tasks.push(t); } workers.clear(); for (int i = 0; i < settings.num_threads; i++) { workers.push_back(std::thread(sam_worker, std::ref(sam_tasks), &settings, index)); } for (auto& w : workers) { w.join(); } std::cout << "Total run time: " << time(NULL) - t_start << " s" << std::endl; delete index; return 1; }